vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.mdsal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  14
  15 import com.google.common.base.Preconditions;
  16 import com.google.common.collect.Iterators;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Optional;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PreDestroy;
  38 import javax.inject.Inject;
  39 import javax.inject.Singleton;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  49 import org.opendaylight.genius.mdsalutil.NWUtil;
  50 import org.opendaylight.genius.mdsalutil.NwConstants;
  51 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  52 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  53 import org.opendaylight.genius.utils.JvmGlobalLocks;
  54 import org.opendaylight.infrautils.caches.CacheProvider;
  55 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  56 import org.opendaylight.infrautils.utils.concurrent.Executors;
  57 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  58 import org.opendaylight.mdsal.binding.api.DataBroker;
  59 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  60 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  61 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  62 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  63 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  64 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  65 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  66 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  67 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  68 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  69 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  70 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  71 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  72 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  74 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 118 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 120 import org.opendaylight.yangtools.yang.common.Uint32;
 121 import org.opendaylight.yangtools.yang.common.Uint64;
 122 import org.slf4j.Logger;
 123 import org.slf4j.LoggerFactory;
 124
 125 @Singleton
 126 public class VpnInterfaceManager extends AbstractAsyncDataTreeChangeListener<VpnInterface> {
 127
 128     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 129     private static final short DJC_MAX_RETRIES = 3;
 130
 131     private final DataBroker dataBroker;
 132     private final ManagedNewTransactionRunner txRunner;
 133     private final IBgpManager bgpManager;
 134     private final IFibManager fibManager;
 135     private final IMdsalApiManager mdsalManager;
 136     private final IdManagerService idManager;
 137     private final OdlInterfaceRpcService ifaceMgrRpcService;
 138     private final VpnFootprintService vpnFootprintService;
 139     private final IInterfaceManager interfaceManager;
 140     private final IVpnManager vpnManager;
 141     private final ArpResponderHandler arpResponderHandler;
 142     private final JobCoordinator jobCoordinator;
 143     private final VpnUtil vpnUtil;
 144
 145     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 146
 147     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 148             new ConcurrentHashMap<>();
 149
 150     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 151
 152     @Inject
 153     public VpnInterfaceManager(final DataBroker dataBroker,
 154                                final IBgpManager bgpManager,
 155                                final IdManagerService idManager,
 156                                final IMdsalApiManager mdsalManager,
 157                                final IFibManager fibManager,
 158                                final OdlInterfaceRpcService ifaceMgrRpcService,
 159                                final VpnFootprintService vpnFootprintService,
 160                                final IInterfaceManager interfaceManager,
 161                                final IVpnManager vpnManager,
 162                                final ArpResponderHandler arpResponderHandler,
 163                                final JobCoordinator jobCoordinator,
 164                                final CacheProvider cacheProvider,
 165                                final VpnUtil vpnUtil) {
 166         super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 167                 InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class),
 168                 Executors.newListeningSingleThreadExecutor("VpnInterfaceManager", LOG));
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187         start();
 188     }
 189
 190     public Runnable isNotifyTaskQueued(String intfName) {
 191         return vpnIntfMap.remove(intfName);
 192     }
 193
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196     }
 197
 198     @Override
 199     @PreDestroy
 200     public void close() {
 201         super.close();
 202         vpnInstanceOpDataEntryCache.close();
 203         Executors.shutdownAndAwaitTermination(getExecutorService());
 204     }
 205
 206     @Override
 207     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 208         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 209         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 210                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 211         addVpnInterface(identifier, vpnInterface, null, null);
 212     }
 213
 214     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 215                           final VpnInterface vpnInterface, String vpnName) {
 216         // FIXME: separate this out somehow?
 217         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 218         lock.lock();
 219         try {
 220             if (isVpnInstanceReady(vpnName)) {
 221                 return true;
 222             }
 223             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 224             return false;
 225         } finally {
 226             lock.unlock();
 227         }
 228     }
 229
 230     // TODO Clean up the exception handling
 231     @SuppressWarnings("checkstyle:IllegalCatch")
 232     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 233                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 234         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 235             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 236             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 237         }
 238     }
 239
 240     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 241                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 242         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 243         final String interfaceName = key.getName();
 244
 245         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 246             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 247                   vpnName, vpnInterface.getName());
 248             return;
 249         }
 250         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 251              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 252         List<Adjacency> copyOldAdjs = null;
 253         if (oldAdjs != null) {
 254             copyOldAdjs = new ArrayList<>();
 255             copyOldAdjs.addAll(oldAdjs);
 256         }
 257         List<Adjacency> copyNewAdjs = null;
 258         if (newAdjs != null) {
 259             copyNewAdjs = new ArrayList<>();
 260             copyNewAdjs.addAll(newAdjs);
 261         }
 262         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 263     }
 264
 265     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 266                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 267                     final @Nullable List<Adjacency> newAdjs,
 268                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 269         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 270         final String interfaceName = key.getName();
 271         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 272         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 273             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 274             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 275             if (interfaceState != null) {
 276                 try {
 277                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 278                     final int ifIndex = interfaceState.getIfIndex();
 279                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 280                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 281                         // (the inventory tx goes in last)
 282                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 283                         //set of prefix used, as entry in prefix-to-interface datastore
 284                         // is prerequisite for refresh Fib to avoid race condition leading to
 285                         // missing remote next hop in bucket actions on bgp-vpn delete
 286                         Set<String> prefixListForRefreshFib = new HashSet<>();
 287                         ListenableFuture<Void> confFuture =
 288                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 289                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 290                                     operTx -> futures.add(
 291                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 292                                             LOG.info(
 293                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 294                                                     + " on dpn {}",
 295                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 296                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 297                                                 confTx, operTx, invTx, interfaceState, vpnName,
 298                                                 prefixListForRefreshFib);
 299                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 300                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 301                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 302                                                 if (newAdjs != null) {
 303                                                     for (Adjacency adj : newAdjs) {
 304                                                         if (oldAdjs.contains(adj)) {
 305                                                             oldAdjs.remove(adj);
 306                                                         } else {
 307                                                             if (!isBgpVpnInternetVpn
 308                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 309                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 310                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 311                                                                     prefixListForRefreshFib);
 312                                                             }
 313                                                         }
 314                                                     }
 315                                                 }
 316                                                 for (Adjacency adj : oldAdjs) {
 317                                                     if (!isBgpVpnInternetVpn
 318                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 320                                                             operTx, confTx);
 321                                                     }
 322                                                 }
 323                                             }
 324                                         })))));
 325                         Futures.addCallback(confFuture,
 326                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 327                             MoreExecutors.directExecutor());
 328                         futures.add(confFuture);
 329                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 330                             MoreExecutors.directExecutor());
 331                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 332                             + " processed successfully", interfaceName, vpnName, dpnId);
 333                         return futures;
 334                     });
 335                 } catch (NumberFormatException | IllegalStateException e) {
 336                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 337                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 338                         vpnName, e);
 339                     return;
 340                 }
 341             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 342                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 343                     () -> {
 344                         ListenableFuture<Void> future =
 345                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 346                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 347                                     confTx, vpnName);
 348                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 349                                     vpnName, vpnInterface.getDpnId());
 350                             });
 351                         ListenableFutures.addErrorLogging(future, LOG,
 352                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 353                         return Collections.singletonList(future);
 354                     });
 355             } else {
 356                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 357                     + " is not available", interfaceName, vpnName);
 358             }
 359         } else {
 360             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 361                     + " as vpn is pending delete", interfaceName, vpnName,
 362                 vpnInterface.getDpnId());
 363         }
 364     }
 365
 366     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 367     // see comments below.
 368     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 369     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 370             final int lportTag, boolean isInterfaceUp,
 371             TypedWriteTransaction<Configuration> writeConfigTxn,
 372             TypedWriteTransaction<Operational> writeOperTxn,
 373             TypedReadWriteTransaction<Configuration> writeInvTxn,
 374             Interface interfaceState, final String vpnName,
 375             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 376         final String interfaceName = vpnInterface.getName();
 377         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 378                 vpnName);
 379         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 380         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 381         if (!isInterfaceUp) {
 382             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 383                      interfaceName, dpId, vpnName);
 384             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 385             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 386                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 387                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 388                         dpId);
 389                 return;
 390             }
 391
 392             boolean waitForVpnInterfaceOpRemoval = false;
 393             if (opVpnInterface != null) {
 394                 String opVpnName = opVpnInterface.getVpnInstanceName();
 395                 String primaryInterfaceIp = null;
 396                 if (Objects.equals(opVpnName, vpnName)) {
 397                     // Please check if the primary VRF Entry does not exist for VPNInterface
 398                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 399                     // back to back
 400                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 401                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 402                     if (adjs == null) {
 403                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 404                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 405                                 vpnName);
 406                         return;
 407                     }
 408                     for (Adjacency adj : adjs) {
 409                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 410                             primaryInterfaceIp = adj.getIpAddress();
 411                             break;
 412                         }
 413                     }
 414                     if (primaryInterfaceIp == null) {
 415                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 416                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 417                                 dpId, vpnName);
 418                         return;
 419                     }
 420                     // Get the rd of the vpn instance
 421                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 422                     if (vrf != null) {
 423                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 424                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 425                         return;
 426                     }
 427                     waitForVpnInterfaceOpRemoval = true;
 428                 } else {
 429                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 430                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 431                 }
 432             }
 433             if (!waitForVpnInterfaceOpRemoval) {
 434                 // Add the VPNInterface and quit
 435                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 436                         null/*ipAddressSourceValuePair*/,
 437                         true /* add */);
 438                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 439                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 440                 if (!isBgpVpnInternetVpn) {
 441                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 442                 }
 443                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 444                         dpId, vpnName);
 445                 if (interfaceManager.isExternalInterface(interfaceName)) {
 446                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 447                         NwConstants.ADD_FLOW);
 448                 }
 449                 return;
 450             }
 451
 452             // FIB didn't get a chance yet to clean up this VPNInterface
 453             // Let us give it a chance here !
 454             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 455                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 456             try {
 457                 Runnable notifyTask = new VpnNotifyTask();
 458                 synchronized (notifyTask) {
 459                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 460                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 461                     // published here while holding the lock on it so this path will hit the wait before notify can be
 462                     // invoked.
 463                     vpnIntfMap.put(interfaceName, notifyTask);
 464                     try {
 465                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 466                     } catch (InterruptedException e) {
 467                         // Ignored
 468                     }
 469                 }
 470             } finally {
 471                 vpnIntfMap.remove(interfaceName);
 472             }
 473
 474             if (opVpnInterface != null) {
 475                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 476                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 477                         dpId, vpnName);
 478                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 479                 return;
 480             }
 481             // VPNInterface got removed, proceed with Add
 482             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 483                     interfaceName, dpId, vpnName);
 484             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 485                     null/*ipAddressSourceValuePair*/,
 486                     true /* add */);
 487             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 488                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 489             if (!isBgpVpnInternetVpn) {
 490                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 491             }
 492             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 493                     + " FIB to clean up", interfaceName, dpId, vpnName);
 494             if (interfaceManager.isExternalInterface(interfaceName)) {
 495                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 496                                lportTag, NwConstants.ADD_FLOW);
 497             }
 498         } else {
 499             try {
 500                 // Interface is retained in the DPN, but its Link Up.
 501                 // Advertise prefixes again for this interface to BGP
 502                 InstanceIdentifier<VpnInterface> identifier =
 503                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 504                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 505                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 506                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 507                 // Perform similar operation as interface add event for extraroutes.
 508                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 509                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 510                         LogicalDatastoreType.CONFIGURATION, path);
 511                 if (!optAdjacencies.isPresent()) {
 512                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 513                     return;
 514                 }
 515                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 516                 for (Adjacency adjacency : adjacencies) {
 517                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 518                         continue;
 519                     }
 520                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 521                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 522                         continue;
 523                     }
 524                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 525                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 526                 }
 527             } catch (InterruptedException | ExecutionException e) {
 528                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 529                         interfaceName, vpnName, primaryRd, dpId);
 530             }
 531         }
 532     }
 533
 534     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 535         int lportTag, int addOrRemove) {
 536         Uuid extNetworkId;
 537         try {
 538             // vpn instance of ext-net interface is the network-id
 539             extNetworkId = new Uuid(vpnName);
 540         } catch (IllegalArgumentException e) {
 541             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 542                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 543             return;
 544         }
 545
 546         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 547         if (routerIds == null || routerIds.isEmpty()) {
 548             LOG.info("processExternalVpnInterface: No router is associated with {}."
 549                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 550                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 551             return;
 552         }
 553
 554         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 555                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 556         for (Uuid routerId : routerIds) {
 557             String routerName = routerId.getValue();
 558             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 559             if (Objects.equals(primarySwitch, dpId)) {
 560                 Routers router = vpnUtil.getExternalRouter(routerName);
 561                 if (router != null) {
 562                     if (addOrRemove == NwConstants.ADD_FLOW) {
 563                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 564                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 565                                 dpId, interfaceName, lportTag);
 566                     } else {
 567                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 568                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 569                                 dpId, interfaceName, lportTag);
 570                     }
 571                 } else {
 572                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 573                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 574                             interfaceName, dpId, vpnName);
 575                 }
 576             }
 577         }
 578     }
 579
 580     // TODO Clean up the exception handling
 581     @SuppressWarnings("checkstyle:IllegalCatch")
 582     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 583                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 584                                                   String vpnName, String interfaceName) {
 585         if (rd == null) {
 586             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 587                   interfaceName, dpnId, vpnName);
 588             return;
 589         } else {
 590             if (rd.equals(vpnName)) {
 591                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 592                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 593                 return;
 594             }
 595         }
 596         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 597                 interfaceName, dpnId, vpnName, rd);
 598
 599         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 600         if (nextHopIp == null) {
 601             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 602                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 603                     rd, vpnName);
 604             return;
 605         }
 606
 607         try {
 608             //Read NextHops
 609             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 610             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 611                     LogicalDatastoreType.OPERATIONAL, path);
 612             if (adjacencies.isPresent()) {
 613                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 614                 if (nextHops != null && !nextHops.isEmpty()) {
 615                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 616                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 617                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 618                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 619                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 620                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 621                     for (Adjacency nextHop : nextHops) {
 622                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 623                             continue;
 624                         }
 625                         String gatewayMac = null;
 626                         Uint32 label = Uint32.ZERO;
 627                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 628                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 629                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 630                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 631                                     .get();
 632                         } else {
 633                             label = nextHop.getLabel();
 634                         }
 635                         try {
 636                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 637                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 638                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 639                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 640                                     gatewayMac);
 641                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 642                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 643                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 644                         } catch (Exception e) {
 645                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 646                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 647                                     interfaceName, dpnId, e);
 648                         }
 649                     }
 650                 }
 651             }
 652         } catch (InterruptedException | ExecutionException e) {
 653             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 654                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 655         }
 656     }
 657
 658     // TODO Clean up the exception handling
 659     @SuppressWarnings("checkstyle:IllegalCatch")
 660     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 661                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 662                         TypedWriteTransaction<Operational> writeOperTx) {
 663         //Read NextHops
 664         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 665         String rd = vpnUtil.getVpnRd(interfaceName);
 666         if (rd == null) {
 667             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 668                 interfaceName, vpnName);
 669             return;
 670         } else {
 671             if (rd.equals(vpnName)) {
 672                 LOG.info(
 673                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 674                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 675                 return;
 676             }
 677         }
 678         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 679                vpnName, rd);
 680         Optional<AdjacenciesOp> adjacencies = Optional.empty();
 681         try {
 682             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 683                     path);
 684         } catch (InterruptedException | ExecutionException e) {
 685             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 686                     interfaceName, vpnName);
 687         }
 688         if (adjacencies.isPresent()) {
 689             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 690
 691             if (nextHops != null && !nextHops.isEmpty()) {
 692                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 693                         nextHops, interfaceName, vpnName, rd);
 694                 for (Adjacency nextHop : nextHops) {
 695                     try {
 696                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 697                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 698                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 699                                     interfaceName, vpnName);
 700                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 701                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 702                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 703                                     interfaceName, vpnName);
 704                         } else if (nextHop.getNextHopIpList() != null) {
 705                             // Perform similar operation as interface delete event for extraroutes.
 706                             String allocatedRd = nextHop.getVrfId();
 707                             for (String nh : nextHop.getNextHopIpList()) {
 708                                 deleteExtraRouteFromCurrentAndImportingVpns(
 709                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 710                                     writeOperTx);
 711                             }
 712                         }
 713                     } catch (Exception e) {
 714                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 715                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 716                     }
 717                 }
 718             }
 719         }
 720     }
 721
 722     @SuppressWarnings("checkstyle:IllegalCatch")
 723     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 724                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 725                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 726                                                   TypedWriteTransaction<Operational> writeOperTxn,
 727                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 728                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 729             throws ExecutionException, InterruptedException {
 730         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 731         // Read NextHops
 732         Optional<VpnInterface> vpnInteface = Optional.empty();
 733         try {
 734             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 735             LogicalDatastoreType.CONFIGURATION, identifier);
 736         } catch (InterruptedException | ExecutionException e) {
 737             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 738                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 739         }
 740         Uuid intfnetworkUuid = null;
 741         NetworkType networkType = null;
 742         Long segmentationId = Long.valueOf(-1);
 743         Adjacencies adjacencies = null;
 744         if (vpnInteface.isPresent()) {
 745             intfnetworkUuid = vpnInteface.get().getNetworkId();
 746             networkType = vpnInteface.get().getNetworkType();
 747             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 748             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 749             if (adjacencies == null) {
 750                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 751                         null/*gwMac*/,  null/*gatewayIp*/, writeOperTxn);
 752                 return;
 753             }
 754         }
 755         // Get the rd of the vpn instance
 756         String nextHopIp = null;
 757         String gatewayIp = null;
 758         try {
 759             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 760         } catch (Exception e) {
 761             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 762                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 763         }
 764         List<String> nhList = new ArrayList<>();
 765         if (nextHopIp != null) {
 766             nhList.add(nextHopIp);
 767             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 768                     interfaceName, dpnId, vpnName, nhList);
 769         }
 770         Optional<String> gwMac = Optional.empty();
 771         String vpnInterfaceSubnetGwMacAddress = null;
 772         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 773         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 774         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 775         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 776         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 777         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 778         List<Adjacency> value = new ArrayList<>();
 779         for (Adjacency nextHop : nextHops) {
 780             String rd = primaryRd;
 781             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 782             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 783                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 784                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 785                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 786                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 787                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 788                 continue;
 789             }
 790             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 791                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 792                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 793                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 794                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 795                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 796
 797                 Prefixes prefixes = intfnetworkUuid != null
 798                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 799                             segmentationId, prefixCue) :
 800                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 801                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 802                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 803                 final Uuid subnetId = nextHop.getSubnetId();
 804
 805                 gatewayIp = nextHop.getSubnetGatewayIp();
 806                 if (gatewayIp == null) {
 807                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 808                     if (gatewayIpOptional.isPresent()) {
 809                         gatewayIp = gatewayIpOptional.get();
 810                     }
 811                 }
 812
 813                 if (gatewayIp != null) {
 814                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 815                     if (gwMac.isPresent()) {
 816                         // A valid mac-address is available for this subnet-gateway-ip
 817                         // Use this for programming ARP_RESPONDER table here.  And save this
 818                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 819                         // to populate L3_GW_MAC_TABLE there.
 820                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 821                                 gatewayIp, gwMac.get());
 822                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 823                     } else {
 824                         // A valid mac-address is not available for this subnet-gateway-ip
 825                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 826                         // Save this connected-mac-address as gateway-mac-address for the
 827                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 828                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 829                         if (gwMac.isPresent()) {
 830                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 831                                     NwConstants.ADD_FLOW, gwMac.get());
 832                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 833                                     gatewayIp, gwMac.get());
 834                         } else {
 835                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 836                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 837                                 + "gwIp {}",
 838                                 subnetId, interfaceName, vpnName, gatewayIp);
 839                         }
 840                     }
 841                 } else {
 842                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 843                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 844                         subnetId, interfaceName, vpnName);
 845                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 846                 }
 847                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 848                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 849             } else {
 850                 //Extra route adjacency
 851                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 852                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 853                 // FIXME: separate this out somehow?
 854                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 855                 lock.lock();
 856                 try {
 857                     java.util.Optional<String> rdToAllocate = vpnUtil
 858                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 859                                     nextHop.getNextHopIpList().get(0), dpnId);
 860                     if (rdToAllocate.isPresent()) {
 861                         rd = rdToAllocate.get();
 862                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 863                             rd, prefix);
 864                     } else {
 865                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 866                         continue;
 867                     }
 868                 } finally {
 869                     lock.unlock();
 870                 }
 871                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 872                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 873                         interfaceName, dpnId);
 874             }
 875             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 876             // can be different from the gateway-mac-address within the VRFEntry as the
 877             // gateway-mac-address is a superset.
 878             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 879             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 880                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 881                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 882             Adjacency operationalAdjacency = null;
 883             try {
 884                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 885             } catch (NullPointerException e) {
 886                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 887                     input, e.getMessage());
 888                 return;
 889             }
 890             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 891                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 892                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 893                     writeConfigTxn);
 894             }
 895             value.add(operationalAdjacency);
 896         }
 897
 898         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 899         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 900                 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
 901
 902         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 903                 .setGatewayMac(gwMac.orElse(null)).setInterfaceName(interfaceName)
 904                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 905
 906         for (Adjacency nextHop : aug.getAdjacency()) {
 907             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 908             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 909                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 910                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 911                 registeredPopulator.populateFib(input, writeConfigTxn);
 912             }
 913         }
 914     }
 915
 916     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 917                                               long lportTag, String gwMac, String gwIp,
 918                                               TypedWriteTransaction<Operational> writeOperTxn) {
 919         VpnInterfaceOpDataEntry opInterface =
 920               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
 921         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 922             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 923         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 924         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 925                 interfaceName, dpnId, vpnName);
 926     }
 927
 928     // TODO Clean up the exception handling
 929     @SuppressWarnings("checkstyle:IllegalCatch")
 930     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 931                                            StateTunnelList stateTunnelList,
 932                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 933                                            TypedWriteTransaction<Operational> writeOperTxn) {
 934
 935         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 936         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 937         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 938         List<Adjacency> adjList =
 939             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 940         if (adjList.isEmpty()) {
 941             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 942                     vpnInterface, srcDpnId);
 943             return;
 944         }
 945         String prefix = null;
 946         List<Adjacency> value = new ArrayList<>();
 947         boolean isFibNextHopAddReqd = false;
 948         String vpnName = vpnInterface.getVpnInstanceName();
 949         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 950         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 951         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 952                 vpnInterface.getName(), vpnInterface.getDpnId(),
 953                 vpnInterface.getVpnInstanceName(), adjList);
 954         for (Adjacency adj : adjList) {
 955             String rd = adj.getVrfId();
 956             rd = rd != null ? rd : vpnName;
 957             prefix = adj.getIpAddress();
 958             Uint32 label = adj.getLabel();
 959             List<String> nhList = Collections.singletonList(srcTepIp);
 960             List<String> nextHopList = adj.getNextHopIpList();
 961             // If TEP is added , update the nexthop of primary adjacency.
 962             // Secondary adj nexthop is already pointing to primary adj IP address.
 963             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 964                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 965                 if (nextHopList != null && !nextHopList.isEmpty()) {
 966                     /* everything right already */
 967                 } else {
 968                     isFibNextHopAddReqd = true;
 969                 }
 970             } else {
 971                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 972                 if (!vrfEntryOptional.isPresent()) {
 973                     continue;
 974                 }
 975                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 976                 if (!nhList.contains(srcTepIp)) {
 977                     nhList.add(srcTepIp);
 978                     isFibNextHopAddReqd = true;
 979                 }
 980                 value.add(adj);
 981             }
 982
 983             if (isFibNextHopAddReqd) {
 984                 updateLabelMapper(label, nhList);
 985                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 986                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 987                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 988                 // Update the VRF entry with nextHop
 989                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, true,
 990                         writeConfigTxn);
 991
 992                 //Get the list of VPN's importing this route(prefix) .
 993                 // Then update the VRF entry with nhList
 994                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 995                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 996                     String vpnRd = vpn.getVrfId();
 997                     if (vpnRd != null) {
 998                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, true,
 999                                 writeConfigTxn);
1000                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1001                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1002                             vpnInterface.getName(), srcDpnId, vpnName,
1003                             vpn.getVpnInstanceName(), vpnRd);
1004                     }
1005                 }
1006                 // Advertise the prefix to BGP only for external vpn
1007                 // since there is a nexthop change.
1008                 try {
1009                     if (!rd.equalsIgnoreCase(vpnName)) {
1010                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1011                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1012                                 null /*gatewayMacAddress*/);
1013                     }
1014                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1015                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1016                             srcDpnId, vpnName);
1017                 } catch (Exception ex) {
1018                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1019                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1020                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1021                 }
1022             }
1023         }
1024         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1025         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1026                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1027                 .addAugmentation(AdjacenciesOp.class, aug).build();
1028         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1029                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1030         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1031         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1032                 vpnInterface.getName(), srcDpnId, vpnName);
1033
1034     }
1035
1036     // TODO Clean up the exception handling
1037     @SuppressWarnings("checkstyle:IllegalCatch")
1038     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1039                                               StateTunnelList stateTunnelList,
1040                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1041                                               TypedWriteTransaction<Operational> writeOperTxn) {
1042
1043         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1044         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1045         String prefix = null;
1046         boolean isNextHopRemoveReqd = false;
1047         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1048         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1049         String vpnName = vpnInterface.getVpnInstanceName();
1050         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1051         String primaryRd = vpnUtil.getVpnRd(vpnName);
1052         if (adjList != null) {
1053             List<Adjacency> value = new ArrayList<>();
1054             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1055                     vpnInterface.getName(), vpnInterface.getDpnId(),
1056                     vpnInterface.getVpnInstanceName(), adjList);
1057             for (Adjacency adj : adjList) {
1058                 List<String> nhList = new ArrayList<>();
1059                 String rd = adj.getVrfId();
1060                 rd = rd != null ? rd : vpnName;
1061                 prefix = adj.getIpAddress();
1062                 List<String> nextHopList = adj.getNextHopIpList();
1063                 Uint32 label = adj.getLabel();
1064                 if (nextHopList != null && !nextHopList.isEmpty()) {
1065                     isNextHopRemoveReqd = true;
1066                 }
1067                 // If TEP is deleted , remove the nexthop from primary adjacency.
1068                 // Secondary adj nexthop will continue to point to primary adj IP address.
1069                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1070                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1071                 } else {
1072                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1073                     if (!vrfEntryOptional.isPresent()) {
1074                         continue;
1075                     }
1076                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1077                     if (nhList.contains(srcTepIp)) {
1078                         nhList.remove(srcTepIp);
1079                         isNextHopRemoveReqd = true;
1080                     }
1081                     value.add(adj);
1082                 }
1083
1084                 if (isNextHopRemoveReqd) {
1085                     updateLabelMapper(label, nhList);
1086                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1087                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1088                             prefix, nhList, vpnName,
1089                             vpnId, rd, vpnInterface.getName());
1090                     // Update the VRF entry with removed nextHop
1091                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, false,
1092                             writeConfigTxn);
1093
1094                     //Get the list of VPN's importing this route(prefix) .
1095                     // Then update the VRF entry with nhList
1096                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1097                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1098                         String vpnRd = vpn.getVrfId();
1099                         if (vpnRd != null) {
1100                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, false,
1101                                     writeConfigTxn);
1102                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1103                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1104                                     nhList, label, vpnInterface.getName(), srcDpnId,
1105                                     vpnName,
1106                                     vpn.getVpnInstanceName(), vpnRd);
1107                         }
1108                     }
1109
1110                     // Withdraw prefix from BGP only for external vpn.
1111                     try {
1112                         if (!rd.equalsIgnoreCase(vpnName)) {
1113                             bgpManager.withdrawPrefix(rd, prefix);
1114                         }
1115                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1116                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1117                                 vpnInterface.getName(), srcDpnId,
1118                                 vpnName);
1119                     } catch (Exception ex) {
1120                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1121                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1122                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1123                     }
1124                 }
1125             }
1126             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1127             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1128                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1129                     .addAugmentation(AdjacenciesOp.class, aug).build();
1130             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1131                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1132             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1133             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1134                          vpnInterface.getName(), srcDpnId, vpnName);
1135         }
1136     }
1137
1138     @SuppressWarnings("checkstyle:IllegalCatch")
1139     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1140         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1141         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry;
1142         String vpnRd = vpnUtil.getVpnRd(vpnName);
1143         try {
1144             VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1145             if (opDataEntry == null) {
1146                 LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}"
1147                         + " when check for vpns exporting the routes", vpnName, vpnRd);
1148                 return vpnsToExportRoute;
1149             }
1150             vpnInstanceOpDataEntry = opDataEntry;
1151         } catch (Exception re) {
1152             LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}"
1153                     + " to check for vpns exporting the routes", vpnName, vpnRd, re);
1154             return vpnsToExportRoute;
1155         }
1156         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1157             if (input.getVpnInstanceName() == null) {
1158                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1159                         input.getVrfId());
1160                 return false;
1161             }
1162             return !input.getVpnInstanceName().equals(vpnName);
1163         };
1164
1165         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1166             Iterable<String> commonRTs =
1167                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1168                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1169             return Iterators.size(commonRTs.iterator()) > 0;
1170         };
1171
1172         vpnsToExportRoute =
1173                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1174                         Collectors.toList());
1175         return vpnsToExportRoute;
1176     }
1177
1178     // TODO Clean up the exception handling
1179     @SuppressWarnings("checkstyle:IllegalCatch")
1180     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1181         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1182         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1183             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1184             if (vrfEntries != null) {
1185                 ListenableFutures.addErrorLogging(
1186                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1187                         for (VrfEntry vrfEntry : vrfEntries) {
1188                             try {
1189                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1190                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1191                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1192                                             + " is not a controller managed non intervpn link route. Ignoring.",
1193                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1194                                     continue;
1195                                 }
1196                                 String prefix = vrfEntry.getDestPrefix();
1197                                 String gwMac = vrfEntry.getGatewayMacAddress();
1198                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1199                                     String nh = routePath.getNexthopAddress();
1200                                     Uint32 label = routePath.getLabel();
1201                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1202                                         vrfEntry.getOrigin()))) {
1203                                         LOG.info(
1204                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1205                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1206                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1207                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1208                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1209                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1210                                             confTx);
1211                                     } else {
1212                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1213                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1214                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1215                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1216                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1217                                             confTx);
1218                                     }
1219                                 });
1220                             } catch (RuntimeException e) {
1221                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1222                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1223                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1224                             }
1225                         }
1226                     }), LOG, "Error handing VPN exporting routes");
1227             } else {
1228                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1229                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1230             }
1231         }
1232     }
1233
1234     @Override
1235     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1236         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1237         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1238         final String interfaceName = key.getName();
1239         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1240             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1241             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1242         }
1243     }
1244
1245     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1246                                 final VpnInterface vpnInterface, final String vpnName,
1247                                 final String interfaceName) {
1248         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1249             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1250                 ListenableFuture<Void> future =
1251                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1252                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1253                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1254                     });
1255                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1256                         vpnInterface.getName(), vpnName);
1257                 return Collections.singletonList(future);
1258             }, DJC_MAX_RETRIES);
1259         } else {
1260             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1261             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1262         }
1263     }
1264
1265     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1266     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1267                                            final VpnInterface vpnInterface, final String vpnName,
1268                                            final String interfaceName, final Interface interfaceState) {
1269         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1270                 vpnName, vpnInterface.getDpnId());
1271         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1272         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1273             () -> {
1274                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1275                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1276                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1277                         writeOperTxn -> futures.add(
1278                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1279                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1280                                         interfaceName, vpnName);
1281                                 Uint64 dpId;
1282                                 int ifIndex;
1283                                 String gwMacAddress;
1284                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1285                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1286                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1287                                 try {
1288                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1289                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1290                                 } catch (InterruptedException | ExecutionException e) {
1291                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1292                                             interfaceName, vpnName);
1293                                     return;
1294                                 }
1295                                 if (interfaceState != null) {
1296                                     try {
1297                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1298                                     } catch (NumberFormatException | IllegalStateException e) {
1299                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1300                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1301                                                         + " from vpn interface op data store. ", interfaceName,
1302                                                 vpnInterface.getDpnId(), vpnName, e);
1303                                         dpId = Uint64.ZERO;
1304                                     }
1305                                     ifIndex = interfaceState.getIfIndex();
1306                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1307                                 } else {
1308                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1309                                             + " from vpn interface op.", interfaceName);
1310                                     if (optVpnInterface.isPresent()) {
1311                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1312                                         dpId = vpnOpInterface.getDpnId();
1313                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1314                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1315                                     } else {
1316                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1317                                                 + " as interfaceState and vpn interface op is not"
1318                                                 + " available", interfaceName, vpnName);
1319                                         return;
1320                                     }
1321                                 }
1322                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1323                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1324                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1325                                 LOG.info(
1326                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1327                                                 + "successfully",
1328                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1329                             })))));
1330                 futures.add(configFuture);
1331                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1332                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1333                 return futures;
1334             }, DJC_MAX_RETRIES);
1335     }
1336
1337     protected void processVpnInterfaceDown(Uint64 dpId,
1338                                            String interfaceName,
1339                                            int lportTag,
1340                                            String gwMac,
1341                                            VpnInterfaceOpDataEntry vpnOpInterface,
1342                                            boolean isInterfaceStateDown,
1343                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1344                                            TypedWriteTransaction<Operational> writeOperTxn,
1345                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1346             throws ExecutionException, InterruptedException {
1347         if (vpnOpInterface == null) {
1348             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1349                     + " as it is not available in operational data store", interfaceName, dpId);
1350             return;
1351         }
1352         final String vpnName = vpnOpInterface.getVpnInstanceName();
1353         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1354                                                     interfaceName, vpnName);
1355         if (!isInterfaceStateDown) {
1356             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1357             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1358             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1359             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1360                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1361             if (interfaceManager.isExternalInterface(interfaceName)) {
1362                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1363                     NwConstants.DEL_FLOW);
1364             }
1365             if (!isBgpVpnInternetVpn) {
1366                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1367             }
1368             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1369                     + " successful", interfaceName, dpId, vpnName);
1370         } else {
1371             // Interface is retained in the DPN, but its Link Down.
1372             // Only withdraw the prefixes for this interface from BGP
1373             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1374         }
1375     }
1376
1377     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1378                                           final String vpnName, final Uint32 vpnId, String gwMac,
1379                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1380                                           TypedWriteTransaction<Operational> writeOperTxn,
1381                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1382             throws ExecutionException, InterruptedException {
1383         //Read NextHops
1384         try {
1385             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1386                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1387             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1388                     SingleTransactionDataBroker.syncReadOptional(dataBroker,
1389                             LogicalDatastoreType.OPERATIONAL, identifier);
1390             boolean isNonPrimaryAdjIp = false;
1391             String primaryRd = vpnUtil.getVpnRd(vpnName);
1392             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1393                     interfaceName, dpnId, vpnName, primaryRd);
1394             if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1395                 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1396                         + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1397                 return;
1398             }
1399             AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1400
1401             if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) {
1402                 List<Adjacency> nextHops = adjacencies.getAdjacency();
1403                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1404                         interfaceName, dpnId, vpnName, nextHops);
1405                 for (Adjacency nextHop : nextHops) {
1406                     if (nextHop.isPhysNetworkFunc()) {
1407                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1408                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1409                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1410                                 null, null/*writeCfgTxn*/);
1411                     } else {
1412                         String rd = nextHop.getVrfId();
1413                         List<String> nhList;
1414                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1415                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1416                             isNonPrimaryAdjIp = Boolean.TRUE;
1417                         } else {
1418                             // This is a primary adjacency
1419                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1420                                     : emptyList();
1421                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1422                                     vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1423                                     interfaceName, writeInvTxn);
1424                             isNonPrimaryAdjIp = Boolean.FALSE;
1425                         }
1426                         if (!nhList.isEmpty()) {
1427                             if (Objects.equals(primaryRd, vpnName)) {
1428                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1429                                 //remove from FIB directly
1430                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1431                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1432                             } else {
1433                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1434                                         interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn);
1435                             }
1436                         } else {
1437                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1438                                             + " interface {}", nextHop.getIpAddress(), rd,
1439                                     nextHop.getAdjacencyType().toString(), interfaceName);
1440                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1441                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1442                         }
1443                     }
1444                     String ip = nextHop.getIpAddress().split("/")[0];
1445                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1446                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1447                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1448                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1449                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1450                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1451                     }
1452                     // Remove the MIP-IP from VpnPortIpToPort.
1453                     if (isNonPrimaryAdjIp) {
1454                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1455                         if (persistedIp != null && persistedIp.isLearntIp()
1456                                 && persistedIp.getPortName().equals(interfaceName)) {
1457                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1458                             LOG.info(
1459                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1460                                             + "from VpnPortipToPort",
1461                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1462                         }
1463                     }
1464                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1465                     if (vpnPortipToPort != null) {
1466                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1467                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1468                                  + "Interface {} ip {} on dpn {} for vpn {}",
1469                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1470                     }
1471                 }
1472             } else {
1473                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1474                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1475                         + " Removing it.", interfaceName, vpnName, dpnId);
1476                 writeOperTxn.delete(identifier);
1477             }
1478         } catch (InterruptedException | ExecutionException e) {
1479             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1480                     interfaceName, dpnId, vpnName);
1481         }
1482     }
1483
1484     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1485                                                             String interfaceName, Uint64 dpnId,
1486                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1487                                                             TypedWriteTransaction<Operational> writeOperTx) {
1488         return (nh) -> {
1489             String primaryRd = vpnUtil.getVpnRd(vpnName);
1490             String prefix = nextHop.getIpAddress();
1491             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1492             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1493                     nextHop, vpnName, interfaceName, dpnId);
1494             // FIXME: separate this out somehow?
1495             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1496             lock.lock();
1497             try {
1498                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1499                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1500                     //If extra-route is present behind at least one VM, then do not remove or update
1501                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1502                     //prefixtointerface DS
1503                     return;
1504                 }
1505                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1506                         writeConfigTxn);
1507             } finally {
1508                 lock.unlock();
1509             }
1510             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1511                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1512                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1513         };
1514     }
1515
1516     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1517                                            Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp,
1518                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1519                                            TypedWriteTransaction<Operational> writeOperTx) {
1520         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1521                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1522         nhList.forEach((nh) -> {
1523             //IRT: remove routes from other vpns importing it
1524             if (isNonPrimaryAdjIp) {
1525                 removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1526             } else {
1527                 vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1528                         nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1529             }
1530             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1531                 String vpnRd = vpn.getVrfId();
1532                 if (vpnRd != null) {
1533                     fibManager.removeOrUpdateFibEntry(vpnRd,
1534                             nextHop.getIpAddress(), nh, writeConfigTxn);
1535                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1536                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1537                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1538                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1539                 }
1540             }
1541         });
1542     }
1543
1544     @SuppressWarnings("checkstyle:IllegalCatch")
1545     private void removeLearntPrefixFromBGP(String rd, String prefix, String nextHop,
1546                                            TypedWriteTransaction<Configuration> writeConfigTxn) {
1547         try {
1548             if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) {
1549                 LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this"
1550                         + " operation", prefix, nextHop, rd);
1551                 return;
1552             }
1553             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}",
1554                     rd, prefix, nextHop);
1555             fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn);
1556             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1557             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}",
1558                     rd, prefix, nextHop);
1559         } catch (Exception e) {
1560             LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e);
1561         }
1562     }
1563
1564     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1565                                                  int lportTag, String gwMac, String gwIp, String interfaceName,
1566                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1567             throws ExecutionException, InterruptedException {
1568         final Uuid subnetId = nextHop.getSubnetId();
1569         if (nextHop.getSubnetGatewayMacAddress() == null) {
1570             // A valid mac-address was not available for this subnet-gateway-ip
1571             // So a connected-mac-address was used for this subnet and we need
1572             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1573             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1574         }
1575         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1576                 subnetId);
1577     }
1578
1579     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1580                                                   String interfaceName) {
1581         // This is either an extra-route (or) a learned IP via subnet-route
1582         List<String> nhList = null;
1583         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1584         if (nextHopIp == null || nextHopIp.isEmpty()) {
1585             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1586                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1587                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1588                     vpnName);
1589             nhList = emptyList();
1590         } else {
1591             nhList = Collections.singletonList(nextHopIp);
1592         }
1593         return nhList;
1594     }
1595
1596     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1597         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1598         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1599         // else use connected interface
1600         if (gwPort != null && gwPort.isSubnetIp()) {
1601             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1602                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1603             return Optional.of(gwPort.getMacAddress());
1604         }
1605         return Optional.empty();
1606     }
1607
1608     @Override
1609     public void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1610         final VpnInterface update) {
1611         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1612         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1613                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1614         if (original.equals(update)) {
1615             LOG.info("update: original {} update {} are same. No update required.", original, update);
1616             return;
1617         }
1618         final String vpnInterfaceName = update.getName();
1619         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1620         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1621         //handles switching between <internal VPN - external VPN>
1622         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1623             List<ListenableFuture<Void>> futures = new ArrayList<>();
1624             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1625                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1626                                 + "to newVpn(s) {}",
1627                         original.getName(), dpnId,
1628                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1629                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1630                 return emptyList();
1631             }
1632             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1633             return futures;
1634         });
1635     }
1636
1637     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1638                                                            VpnInterface original, VpnInterface update,
1639                                                            List<ListenableFuture<Void>> futures) {
1640         boolean isVpnInstanceUpdate = false;
1641         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1642         final String interfaceName = key.getName();
1643         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1644         List<String> oldVpnListCopy = new ArrayList<>();
1645         oldVpnListCopy.addAll(oldVpnList);
1646         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1647         List<String> newVpnListCopy = new ArrayList<>();
1648         newVpnListCopy.addAll(newVpnList);
1649
1650         oldVpnList.removeAll(newVpnList);
1651         newVpnList.removeAll(oldVpnListCopy);
1652         //This block will execute only on if there is a change in the VPN Instance.
1653         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1654             /*
1655              * Internet BGP-VPN Instance update with single router:
1656              * ====================================================
1657              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1658              *     1st VPN Instance : router VPN or external BGP-VPN.
1659              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1660              *
1661              * VPN Instance UPDATE:
1662              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1663              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1664              *
1665              * External BGP-VPN Instance update with single router:
1666              * ====================================================
1667              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1668              *
1669              * Updated VPN Instance will be always either internal router VPN to
1670              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1671              *
1672              * VPN Instance UPDATE:
1673              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1674              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1675              *
1676              * Dual Router VPN Instance Update:
1677              * ================================
1678              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1679              *
1680              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1681              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1682              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1683              *
1684              * Dual Router --> Associated with common external BGP-VPN Instance.
1685              * 1st router and 2nd router are getting associated with single External BGP-VPN
1686              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1687              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1688              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1689              *
1690              *
1691              */
1692             isVpnInstanceUpdate = true;
1693             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1694                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1695                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1696                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1697                     List<String> externalBgpVpnList = new ArrayList<>();
1698                     for (String newVpnName : newVpnListCopy) {
1699                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1700                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1701                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1702                                 .BgpvpnType.BGPVPNExternal) {
1703                             externalBgpVpnList.add(newVpnName);
1704                             break;
1705                         }
1706                     }
1707                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1708                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1709                             externalBgpVpnList, oldVpnListCopy, futures);
1710
1711                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1712                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1713                     //Identify the router VPN Instance and pass that value as oldVpnList
1714                     List<String> routerVpnList = new ArrayList<>();
1715                     for (String newVpnName : newVpnListCopy) {
1716                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1717                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1718                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1719                                 .BgpvpnType.VPN) {
1720                             routerVpnList.add(newVpnName);
1721                             break;
1722                         }
1723                     }
1724                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1725                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1726                             newVpnList, oldVpnListCopy, futures);
1727
1728                 } else {
1729                     //Handle remaining use cases.
1730                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1731                             oldVpnListCopy, futures);
1732                 }
1733             } else {
1734                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1735                         oldVpnListCopy, futures);
1736             }
1737         }
1738         return isVpnInstanceUpdate;
1739     }
1740
1741     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1742                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1743                                          List<String> newVpnList, List<String> oldVpnListCopy,
1744                                          List<ListenableFuture<Void>> futures) {
1745         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1746         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1747                 ? origAdjs.getAdjacency() : new ArrayList<>();
1748         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1749         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1750                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1751
1752         boolean isOldVpnRemoveCallExecuted = false;
1753         for (String oldVpnName : oldVpnList) {
1754             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1755                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1756             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1757             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1758                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1759                     newVpnList);
1760             isOldVpnRemoveCallExecuted = true;
1761         }
1762         //Wait for previous interface bindings to be removed
1763         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1764             try {
1765                 Thread.sleep(2000);
1766             } catch (InterruptedException e) {
1767                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1768             }
1769         }
1770         for (String newVpnName : newVpnList) {
1771             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1772             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1773                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1774                         + "onto vpnName {} ", interfaceName, newVpnName);
1775                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1776                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1777                                 + "from oldVpn(s) {} to newVpn {} ",
1778                         interfaceName, oldVpnListCopy, newVpnName);
1779                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1780                  * Use Case:
1781                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1782                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1783                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1784                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1785                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1786                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1787                  *     router VPN as well as Internet BGP-VPN.
1788                  *
1789                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1790                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1791                  *                                       update in existing router VPN instance.
1792                  */
1793                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1794                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1795                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1796                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1797                 }
1798             } else {
1799                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1800                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1801                         interfaceName, oldVpnListCopy, newVpnName);
1802             }
1803         }
1804     }
1805
1806     // TODO Clean up the exception handling
1807     @SuppressWarnings("checkstyle:IllegalCatch")
1808     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1809                                                                     String vpnInterfaceName,
1810                                                                     List<ListenableFuture<Void>> futures) {
1811         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1812         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1813                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1814         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1815         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1816                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1817
1818         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1819         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1820             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1821             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1822             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1823             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1824             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1825                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1826                 //set of prefix used as entry in prefix-to-interface datastore
1827                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1828                 // in bucket actions on bgp-vpn delete
1829                 Set<String> prefixListForRefreshFib = new HashSet<>();
1830                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1831                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1832                         operTx -> {
1833                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1834                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1835                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1836                                     update.getName(), newVpnName);
1837                             //handle both addition and removal of adjacencies
1838                             // currently, new adjacency may be an extra route
1839                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1840                             if (!oldAdjs.equals(newAdjs)) {
1841                                 for (Adjacency adj : copyNewAdjs) {
1842                                     if (copyOldAdjs.contains(adj)) {
1843                                         copyOldAdjs.remove(adj);
1844                                     } else {
1845                                         // add new adjacency
1846                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1847                                             try {
1848                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1849                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1850                                             } catch (RuntimeException e) {
1851                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1852                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1853                                             }
1854                                         }
1855                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1856                                             + " added to vpn interface {} on vpn {} dpnId {}",
1857                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1858                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1859                                     }
1860                                 }
1861                                 for (Adjacency adj : copyOldAdjs) {
1862                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1863                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1864                                             && !adj.isPhysNetworkFunc()) {
1865                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1866                                                 confTx);
1867                                             //remove FIB entry
1868                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1869                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1870                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1871                                             //remove BGP entry
1872                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1873                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1874                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1875                                             }
1876                                         } else {
1877                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1878                                                 operTx, confTx);
1879                                         }
1880                                     }
1881                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1882                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1883                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1884                                 }
1885                             }
1886                         })));
1887                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1888                     MoreExecutors.directExecutor());
1889                 futures.add(configTxFuture);
1890                 for (ListenableFuture<Void> future : futures) {
1891                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1892                             update.getName(), update.getVpnInstanceNames());
1893                 }
1894             } else {
1895                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1896                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1897             }
1898         }
1899         return futures;
1900     }
1901
1902     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1903         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1904                 .toString();
1905         // FIXME: separate this out somehow?
1906         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1907         lock.lock();
1908         try {
1909             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1910                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1911             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1912                 LogicalDatastoreType.OPERATIONAL, lriIid);
1913             if (opResult.isPresent()) {
1914                 LabelRouteInfo labelRouteInfo =
1915                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1916                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1917                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1918             }
1919             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1920                     nextHopIpList);
1921         } catch (InterruptedException | ExecutionException e) {
1922             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1923                     nextHopIpList);
1924         } catch (TransactionCommitFailedException e) {
1925             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1926                     nextHopIpList);
1927         } finally {
1928             lock.unlock();
1929         }
1930     }
1931
1932     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1933         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1934
1935         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1936         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1937                 .addAugmentation(SubnetRoute.class, route).build();
1938         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1939         InstanceIdentifierBuilder<VrfTables> idBuilder =
1940             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1941         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1942         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1943         if (writeConfigTxn != null) {
1944             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1945         } else {
1946             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1947         }
1948         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1949                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1950     }
1951
1952     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1953                                            Adjacency adj, Uint64 dpnId,
1954                                            TypedWriteTransaction<Operational> writeOperTxn,
1955                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1956                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1957                                            Set<String> prefixListForRefreshFib)
1958             throws ExecutionException, InterruptedException {
1959         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1960         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1961         try {
1962             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1963                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1964             if (optVpnInterface.isPresent()) {
1965                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1966                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1967                 String vpnName = currVpnIntf.getVpnInstanceName();
1968                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1969                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1970                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1971                         LogicalDatastoreType.OPERATIONAL, adjPath);
1972                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1973                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1974                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1975                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1976                 List<Adjacency> adjacencies = new ArrayList<>();
1977                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1978                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1979                 }
1980                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1981                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1982                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1983                 Adjacency operationalAdjacency = null;
1984                 //Handling dual stack neutron port primary adjacency
1985                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1986                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1987                             currVpnIntf.getName(), vpnName);
1988                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1989                             currVpnIntf.getName());
1990                     if (interfaceState != null) {
1991                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1992                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1993                             prefixListForRefreshFib);
1994                     }
1995                 }
1996                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1997                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1998                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1999                             : RouteOrigin.STATIC;
2000                     String nh = adj.getNextHopIpList().get(0);
2001                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
2002                     // FIXME: separate out to somehow?
2003                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
2004                     lock.lock();
2005                     try {
2006                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
2007                                 vpnId, null, prefix, vpnName, nh, dpnId);
2008                         if (rdToAllocate.isPresent()) {
2009                             input.setRd(rdToAllocate.get());
2010                             operationalAdjacency = populator.createOperationalAdjacency(input);
2011                             int label = operationalAdjacency.getLabel().intValue();
2012                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
2013                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
2014                                     currVpnIntf.getName(), operationalAdjacency, encapType,
2015                                     prefixListForRefreshFib, writeConfigTxn);
2016                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
2017                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
2018                                     vpnName, label, currVpnIntf.getName(), dpnId);
2019                         } else {
2020                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2021                                     vpnName, prefix);
2022                             return;
2023                         }
2024                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2025                         // Keeping the MPLS check for now.
2026                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2027                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2028                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2029                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
2030                             vpnsToImportRoute.forEach(vpn -> {
2031                                 if (vpn.getVrfId() != null) {
2032                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2033                                             vpnId, prefix,
2034                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2035                                             .ifPresent(
2036                                                 rds -> vpnManager.addExtraRoute(
2037                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2038                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2039                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2040                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2041                                 }
2042                             });
2043                         }
2044                     } finally {
2045                         lock.unlock();
2046                     }
2047                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2048                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2049                             currVpnIntf.getName(), vpnName);
2050
2051                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2052                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2053                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2054                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2055                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2056                             Prefixes.PrefixCue.PhysNetFunc);
2057                     if (vpnIntefaceConfig.isPresent()) {
2058                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2059                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2060                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2061                     }
2062
2063                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2064
2065                     writeOperTxn.merge(
2066                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2067                                     prefix), pnfPrefix, true);
2068
2069                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2070                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2071                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2072                             RouteOrigin.LOCAL, writeConfigTxn);
2073
2074                     input.setRd(adj.getVrfId());
2075                 }
2076                 if (operationalAdjacency == null) {
2077                     operationalAdjacency = populator.createOperationalAdjacency(input);
2078                 }
2079                 adjacencies.add(operationalAdjacency);
2080                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2081                 VpnInterfaceOpDataEntry newVpnIntf =
2082                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2083                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2084                                 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2085                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2086             }
2087         } catch (InterruptedException | ExecutionException e) {
2088             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2089                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2090         }
2091     }
2092
2093     @Nullable
2094     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2095         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2096         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2097     }
2098
2099     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2100                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2101                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2102         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2103         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2104         try {
2105             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2106                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2107             if (optVpnInterface.isPresent()) {
2108                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2109                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2110                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2111                         LogicalDatastoreType.OPERATIONAL, path);
2112                 if (optAdjacencies.isPresent()) {
2113                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2114
2115                     if (adjacencies != null && !adjacencies.isEmpty()) {
2116                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2117                         for (Adjacency adjacency : adjacencies) {
2118                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2119                                 String rd = adjacency.getVrfId();
2120                                 if (adj.getNextHopIpList() != null) {
2121                                     for (String nh : adj.getNextHopIpList()) {
2122                                         deleteExtraRouteFromCurrentAndImportingVpns(
2123                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2124                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2125                                     }
2126                                 } else if (adj.isPhysNetworkFunc()) {
2127                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2128                                             adj.getIpAddress(), adj.getSubnetId());
2129                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2130                                             null, writeConfigTxn);
2131                                 }
2132                                 break;
2133                             }
2134
2135                         }
2136                     }
2137                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2138                             dpnId, adj.getVrfId());
2139                 } else {
2140                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2141                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2142                 }
2143             }
2144         } catch (InterruptedException | ExecutionException e) {
2145             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2146                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2147         }
2148     }
2149
2150     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2151                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2152                                     TypedWriteTransaction<Operational> writeOperTx) {
2153         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2154                 destination, nextHop, vpnName, intfName, rd);
2155         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2156         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2157         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2158             String vpnRd = vpn.getVrfId();
2159             if (vpnRd != null) {
2160                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2161                         destination, nextHop, vpnName, intfName, vpnRd);
2162                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2163                         writeOperTx);
2164             }
2165         }
2166     }
2167
2168     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2169         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2170             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2171             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2172     }
2173
2174     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2175         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2176             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2177     }
2178
2179     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2180                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2181         if (vpnInterface == null) {
2182             return;
2183         }
2184         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2185         if (adjs == null) {
2186             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2187                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2188             return;
2189         }
2190         for (Adjacency adj : adjs) {
2191             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2192                 String primaryInterfaceIp = adj.getIpAddress();
2193                 String macAddress = adj.getMacAddress();
2194                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2195
2196                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2197                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2198                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
2199                     LOG.error(
2200                             "createFibEntryForRouterInterface: Unable to retrieve label for vpn pool {}, "
2201                                     + "vpninterface {}, vpn {}, rd {}",
2202                             VpnConstants.VPN_IDPOOL_NAME, interfaceName, vpnName, primaryRd);
2203                     return;
2204                 }
2205                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2206                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2207                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2208                         routerInt, label, writeConfigTxn);
2209                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2210                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2211                         macAddress);
2212             } else {
2213                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2214                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2215                         interfaceName, primaryRd, vpnName);
2216             }
2217         }
2218     }
2219
2220     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2221             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2222         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2223         String rd = vpnUtil.getVpnRd(vpnName);
2224         if (adjs != null) {
2225             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2226             for (Adjacency adj : adjsList) {
2227                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2228                     String primaryInterfaceIp = adj.getIpAddress();
2229                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2230                     fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2231                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2232                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2233                 }
2234             }
2235         } else {
2236             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2237                     vpnInterface.getName(), rd);
2238         }
2239     }
2240
2241     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2242         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2243         final String interfaceName = key.getName();
2244         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2245                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2246         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2247                   intefaceData.identifier, vpnName);
2248     }
2249
2250     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2251                                               VpnInterface vpnInterface, String vpnName) {
2252         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2253                .get(vpnName);
2254         if (vpnInterfaces == null) {
2255             vpnInterfaces = new ConcurrentLinkedQueue<>();
2256         }
2257         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2258         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2259         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2260                 vpnInterface.getName(), vpnName);
2261     }
2262
2263     public boolean isVpnInstanceReady(String vpnInstanceName) {
2264         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2265         if (vpnRd == null) {
2266             return false;
2267         }
2268         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2269
2270         return vpnInstanceOpDataEntry != null;
2271     }
2272
2273     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2274         // FIXME: separate out to somehow?
2275         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2276         lock.lock();
2277         try {
2278             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2279                     unprocessedVpnInterfaces.get(vpnInstanceName);
2280             if (vpnInterfaces != null) {
2281                 while (!vpnInterfaces.isEmpty()) {
2282                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2283                     if (hasVpnInstanceCreatedSuccessfully) {
2284                         processSavedInterface(savedInterface, vpnInstanceName);
2285                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2286                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2287                     } else {
2288                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2289                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2290                     }
2291                 }
2292             } else {
2293                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2294             }
2295         } finally {
2296             lock.unlock();
2297         }
2298     }
2299
2300     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2301             VpnInterface vpnInterface) {
2302         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2303         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2304         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2305         lock.lock();
2306         try {
2307             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2308                     unprocessedVpnInterfaces.get(firstVpnName);
2309             if (vpnInterfaces != null) {
2310                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2311                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2312                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2313                 }
2314             } else {
2315                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2316             }
2317         } finally {
2318             lock.unlock();
2319         }
2320     }
2321
2322     public void vpnInstanceIsReady(String vpnInstanceName) {
2323         processSavedInterfaces(vpnInstanceName, true);
2324     }
2325
2326     public void vpnInstanceFailed(String vpnInstanceName) {
2327         processSavedInterfaces(vpnInstanceName, false);
2328     }
2329
2330     private static class UnprocessedVpnInterfaceData {
2331         InstanceIdentifier<VpnInterface> identifier;
2332         VpnInterface vpnInterface;
2333
2334         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2335             this.identifier = identifier;
2336             this.vpnInterface = vpnInterface;
2337         }
2338
2339         @Override
2340         public int hashCode() {
2341             final int prime = 31;
2342             int result = 1;
2343             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2344             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2345             return result;
2346         }
2347
2348         @Override
2349         public boolean equals(Object obj) {
2350             if (this == obj) {
2351                 return true;
2352             }
2353             if (obj == null) {
2354                 return false;
2355             }
2356             if (getClass() != obj.getClass()) {
2357                 return false;
2358             }
2359             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2360             if (identifier == null) {
2361                 if (other.identifier != null) {
2362                     return false;
2363                 }
2364             } else if (!identifier.equals(other.identifier)) {
2365                 return false;
2366             }
2367             if (vpnInterface == null) {
2368                 if (other.vpnInterface != null) {
2369                     return false;
2370                 }
2371             } else if (!vpnInterface.equals(other.vpnInterface)) {
2372                 return false;
2373             }
2374             return true;
2375         }
2376     }
2377
2378     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2379         String primaryRd = vpnUtil.getVpnRd(vpnName);
2380         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2381         if (vpnInstanceOpData == null) {
2382             return;
2383         }
2384         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2385         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2386             return;
2387         }
2388         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2389         vpnToDpnLists.forEach(vpnToDpnList -> {
2390             if (vpnToDpnList.getVpnInterfaces() == null) {
2391                 return;
2392             }
2393             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2394                 try {
2395                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2396                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2397                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2398                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2399                     if (!vpnInterfaceOptional.isPresent()) {
2400                         return;
2401                     }
2402                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2403                             vpnInterface.getInterfaceName());
2404                     if (configVpnAdjacencies == null) {
2405                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2406                         return;
2407                     }
2408                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2409                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2410                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2411                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2412                     // in operational DS. These unprocessed adjacencies will be handled below.
2413                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2414                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2415                     configVpnAdjacencies.stream()
2416                         .filter(adjacency -> operationVpnAdjacencies.stream()
2417                                 .noneMatch(operationalAdjacency ->
2418                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2419                         .forEach(adjacency -> {
2420                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2421                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2422                                 () -> {
2423                                     // TODO Deal with sequencing — the config tx must only submitted
2424                                     // if the oper tx goes in
2425                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2426                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2427                                         futures.add(
2428                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2429                                                 //set of prefix used, as entry in prefix-to-interface datastore
2430                                                 // is prerequisite for refresh Fib to avoid race condition leading
2431                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2432                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2433                                                 ListenableFuture<Void> configTxFuture =
2434                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2435                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2436                                                             primaryRd, adjacency,
2437                                                             vpnInterfaceOptional.get().getDpnId(),
2438                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2439                                                 Futures.addCallback(configTxFuture,
2440                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2441                                                     MoreExecutors.directExecutor());
2442                                                 futures.add(configTxFuture);
2443                                             }));
2444                                         return futures;
2445                                     } else {
2446                                         return emptyList();
2447                                     }
2448                                 });
2449                         });
2450                 } catch (InterruptedException | ExecutionException e) {
2451                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2452                             vpnName, primaryRd);
2453                 }
2454             });
2455         });
2456     }
2457
2458     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2459         private final String interfaceName;
2460         private final boolean add;
2461         private final String txnDestination;
2462
2463         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2464             this.interfaceName = interfaceName;
2465             this.add = add;
2466             this.txnDestination = transactionDest;
2467         }
2468
2469         @Override
2470         public void onSuccess(Void voidObj) {
2471             if (add) {
2472                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2473                         interfaceName, txnDestination);
2474             } else {
2475                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2476             }
2477         }
2478
2479         @Override
2480         public void onFailure(Throwable throwable) {
2481             if (add) {
2482                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2483                         interfaceName, txnDestination, throwable);
2484             } else {
2485                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2486                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2487             }
2488         }
2489     }
2490
2491     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2492         private final String primaryRd;
2493         private final Set<String> prefixListForRefreshFib;
2494
2495         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2496             this.primaryRd = primaryRd;
2497             this.prefixListForRefreshFib = prefixListForRefreshFib;
2498         }
2499
2500         @Override
2501         public void onSuccess(Void voidObj) {
2502             prefixListForRefreshFib.forEach(prefix -> {
2503                 fibManager.refreshVrfEntry(primaryRd, prefix);
2504             });
2505         }
2506
2507         @Override
2508         public void onFailure(Throwable throwable) {
2509             LOG.debug("write Tx config operation failed", throwable);
2510         }
2511     }
2512 }