fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.net.InetAddress;
  25 import java.net.UnknownHostException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.List;
  31 import java.util.Objects;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.locks.ReentrantLock;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.Datastore.Operational;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.TransactionAdapter;
  51 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  52 import org.opendaylight.genius.infra.TypedWriteTransaction;
  53 import org.opendaylight.genius.mdsalutil.ActionInfo;
  54 import org.opendaylight.genius.mdsalutil.FlowEntity;
  55 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  56 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  57 import org.opendaylight.genius.mdsalutil.MatchInfo;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NWUtil;
  60 import org.opendaylight.genius.mdsalutil.NwConstants;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  73 import org.opendaylight.genius.utils.JvmGlobalLocks;
  74 import org.opendaylight.genius.utils.ServiceIndex;
  75 import org.opendaylight.genius.utils.batching.SubTransaction;
  76 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  77 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  78 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  79 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  80 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  81 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  85 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  86 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127
 128 @Singleton
 129 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 130
 131     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 132     private static final String FLOWID_PREFIX = "L3.";
 133     private static final Uint64 COOKIE_VM_FIB_TABLE =  Uint64.valueOf("8000003", 16).intern();
 134     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 135     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 136     private static final int LFIB_INTERVPN_PRIORITY = 15;
 137     public static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 138     private static final int MAX_RETRIES = 3;
 139     private static final Uint64 COOKIE_TABLE_MISS = Uint64.valueOf("8000004", 16).intern();
 140
 141     private final DataBroker dataBroker;
 142     private final ManagedNewTransactionRunner txRunner;
 143     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 144     private final IMdsalApiManager mdsalManager;
 145     private final NexthopManager nextHopManager;
 146     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 147     private final BaseVrfEntryHandler baseVrfEntryHandler;
 148     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 149     private final JobCoordinator jobCoordinator;
 150     private final IElanService elanManager;
 151     private final FibUtil fibUtil;
 152     private final InterVpnLinkCache interVpnLinkCache;
 153     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 154     private final UpgradeState upgradeState;
 155     private final DataTreeEventCallbackRegistrar eventCallbacks;
 156
 157     @Inject
 158     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 159                             final NexthopManager nexthopManager,
 160                             final IElanService elanManager,
 161                             final BaseVrfEntryHandler vrfEntryHandler,
 162                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 163                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 164                             final JobCoordinator jobCoordinator,
 165                             final FibUtil fibUtil,
 166                             final InterVpnLinkCache interVpnLinkCache,
 167                             final UpgradeState upgradeState,
 168                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 169         super(VrfEntry.class, VrfEntryListener.class);
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 173         this.mdsalManager = mdsalApiManager;
 174         this.nextHopManager = nexthopManager;
 175         this.elanManager = elanManager;
 176         this.baseVrfEntryHandler = vrfEntryHandler;
 177         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 178         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 179         this.jobCoordinator = jobCoordinator;
 180         this.fibUtil = fibUtil;
 181         this.interVpnLinkCache = interVpnLinkCache;
 182         this.upgradeState = upgradeState;
 183         this.eventCallbacks = eventCallbacks;
 184     }
 185
 186     @Override
 187     @PostConstruct
 188     public void init() {
 189         LOG.info("{} init", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     @SuppressWarnings("checkstyle:IllegalCatch")
 195     public void close() {
 196         closeables.forEach(c -> {
 197             try {
 198                 c.close();
 199             } catch (Exception e) {
 200                 LOG.warn("Error closing {}", c, e);
 201             }
 202         });
 203     }
 204
 205     @Override
 206     protected VrfEntryListener getDataTreeChangeListener() {
 207         return VrfEntryListener.this;
 208     }
 209
 210     @Override
 211     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 212         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 213     }
 214
 215     @Override
 216     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 217         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 218         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 219         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 220                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 221         addFibEntries(identifier, vrfEntry, rd);
 222         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 223                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 224     }
 225
 226     //This method is temporary. Eventually Factory design pattern will be used to get
 227     // right VrfEntryhandle and invoke its methods.
 228     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 229         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 230             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 234             LOG.info("EVPN flows need to be programmed.");
 235             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 236                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 237             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 238             closeables.add(evpnVrfEntryHandler);
 239             return;
 240         }
 241         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 242         if (routerInt != null) {
 243             // ping responder for router interfaces
 244             routerInterfaceVrfEntryHandler.createFlows(vrfEntry, rd);
 245             return;
 246         }
 247         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 248             createFibEntries(identifier, vrfEntry);
 249             return;
 250         }
 251     }
 252
 253     @Override
 254     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 255         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 256         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 257         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 258                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 259         removeFibEntries(identifier, vrfEntry, rd);
 260         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 261             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 262     }
 263
 264     //This method is temporary. Eventually Factory design pattern will be used to get
 265     // right VrfEntryhandle and invoke its methods.
 266     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 267         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 268             LOG.info("EVPN flows to be deleted");
 269             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 270                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 271             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 272             closeables.add(evpnVrfEntryHandler);
 273             return;
 274         }
 275         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 276         if (routerInt != null) {
 277             // ping responder for router interfaces
 278             routerInterfaceVrfEntryHandler.removeFlows(vrfEntry, rd);
 279             return;
 280         }
 281         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 282             deleteFibEntries(identifier, vrfEntry);
 283             return;
 284         }
 285         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 286             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 287             return;
 288         }
 289     }
 290
 291     @Override
 292     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 293     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 294     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 295     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 296         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 297         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 298         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 299                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 300         // Handle BGP Routes first
 301         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 302             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 303             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 304                     rd, update.getDestPrefix(), update.getRoutePaths());
 305             return;
 306         }
 307
 308         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 309             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 310             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 311             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 312
 313             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 314             //updated vrf entry route path is null. This can happen during tunnel events.
 315             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 316             List<String> usedRds = new ArrayList<>();
 317             if (optVpnInstance.isPresent()) {
 318                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 319                         update.getDestPrefix());
 320             }
 321             // If original VRF Entry had nexthop null , but update VRF Entry
 322             // has nexthop , route needs to be created on remote Dpns
 323             if (originalRoutePath == null || originalRoutePath.isEmpty()
 324                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 325                 // TODO(vivek): Though ugly, Not handling this code now, as each
 326                 // tep add event will invoke flow addition
 327                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 328                         + " This event is IGNORED here.", update.getDestPrefix());
 329                 return;
 330             }
 331
 332             // If original VRF Entry had valid nexthop , but update VRF Entry
 333             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 334             if (updateRoutePath == null || updateRoutePath.isEmpty()
 335                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 336                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 337                         + "This event is IGNORED here.", update.getDestPrefix());
 338                 return;
 339             }
 340             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 341             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 342             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 343             List<ListenableFuture<Void>> futures = new ArrayList<>();
 344             ListenableFuture<Void> configFuture =
 345                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 346                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 347                         nextHopsRemoved.parallelStream()
 348                             .forEach(nextHopRemoved -> {
 349                                 try {
 350                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 351                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 352                                 } catch (ExecutionException | InterruptedException e) {
 353                                     throw new RuntimeException(e);
 354                                 }
 355                             }))));
 356             futures.add(configFuture);
 357             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 358                 @Override
 359                 public void onSuccess(Void result) {
 360                     createFibEntries(identifier, update);
 361                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 362                             rd, update.getDestPrefix(), update.getRoutePaths());
 363                 }
 364
 365                 @Override
 366                 public void onFailure(Throwable throwable) {
 367                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 368                             update, throwable);
 369                 }
 370             }, MoreExecutors.directExecutor());
 371             return;
 372         }
 373
 374         //Handle all other routes only on a cluster reboot
 375         if (original.equals(update)) {
 376             //Reboot use-case
 377             createFibEntries(identifier, update);
 378             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 379                     rd, update.getDestPrefix(), update.getRoutePaths());
 380             return;
 381         }
 382
 383         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 384                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 385     }
 386
 387     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 388         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 389         List<SubTransaction> txnObjects =  new ArrayList<>();
 390         final VpnInstanceOpDataEntry vpnInstance =
 391                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 392         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 393         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 394                 + " has null vpnId!");
 395         final Collection<VpnToDpnList> vpnToDpnList;
 396         if (vrfEntry.getParentVpnRd() != null
 397                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 398             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 399             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 400             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 401                 vpnInstance.getVpnToDpnList();
 402             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 403                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 404         } else {
 405             vpnToDpnList = vpnInstance.getVpnToDpnList();
 406         }
 407         final Uint32 vpnId = vpnInstance.getVpnId();
 408         final String rd = vrfTableKey.getRouteDistinguisher();
 409         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 410         if (subnetRoute != null) {
 411             final long elanTag = subnetRoute.getElantag().toJava();
 412             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 413                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 414             if (vpnToDpnList != null) {
 415                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 416                     () -> Collections.singletonList(
 417                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 418                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 419                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 420                                     installSubnetRouteInFib(curDpn.getDpnId(),
 421                                                                 elanTag, rd, vpnId, vrfEntry, tx);
 422                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
 423                                         vpnId, vrfEntry, NwConstants.ADD_FLOW, tx);
 424                                 }
 425                             }
 426                         })));
 427             }
 428             return;
 429         }
 430         // Get etherType value based on the IpPrefix address family type
 431         int etherType;
 432         try {
 433             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 434         } catch (IllegalArgumentException ex) {
 435             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 436             return;
 437         }
 438
 439         final List<Uint64> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(),
 440                                                                         rd, vrfEntry, etherType);
 441         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 442             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 443                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 444                     final ReentrantLock lock = lockFor(vpnInstance);
 445                     lock.lock();
 446                     try {
 447                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 448                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 449                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 450                                     try {
 451                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 452                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 453                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 454                                                     TransactionAdapter.toWriteTransaction(tx),
 455                                                     txnObjects);
 456                                         } else {
 457                                             createRemoteFibEntry(vpnDpn.getDpnId(),
 458                                                     vpnInstance.getVpnId(),
 459                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 460                                         }
 461                                     } catch (NullPointerException e) {
 462                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 463                                                 vrfEntry.getDestPrefix(), e);
 464                                     }
 465                                 }
 466                             }
 467                         }
 468                     } finally {
 469                         lock.unlock();
 470                     }
 471                 })), MAX_RETRIES);
 472         }
 473
 474         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 475         if (optVpnUuid.isPresent()) {
 476             String vpnUuid = optVpnUuid.get();
 477             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 478             if (interVpnLink != null) {
 479                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 480                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 481                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 482                         // This is an static route that points to the other endpoint of an InterVpnLink
 483                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 484                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 485                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 486                     }
 487                 });
 488             }
 489         }
 490     }
 491
 492     void refreshFibTables(String rd, String prefix) {
 493         InstanceIdentifier<VrfEntry> vrfEntryId =
 494                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 495                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 496         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 497         if (vrfEntry.isPresent()) {
 498             createFibEntries(vrfEntryId, vrfEntry.get());
 499         }
 500     }
 501
 502     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 503         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 504         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 505         prefixBuilder.setDpnId(lri.getDpnId());
 506         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 507         prefixBuilder.setIpAddress(lri.getPrefix());
 508         // Increment the refCount here
 509         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 510             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 511         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 512         if (!isPresentInList) {
 513             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 514             List<String> vpnInstanceNames =
 515                 lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 516             vpnInstanceNames.add(vpnInstanceName);
 517             builder.setVpnInstanceList(vpnInstanceNames);
 518             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 519         } else {
 520             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 521         }
 522         return prefixBuilder.build();
 523     }
 524
 525     void installSubnetRouteInFib(final Uint64 dpnId, final long elanTag, final String rd,
 526             final Uint32 vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 527         if (tx == null) {
 528             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 529                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 530                 "Error installing subnet route in FIB");
 531             return;
 532         }
 533         int etherType;
 534         try {
 535             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 536         } catch (IllegalArgumentException ex) {
 537             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 538             return;
 539         }
 540         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 541             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 542             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 543             final ReentrantLock lock = lockFor(lriKey);
 544             lock.lock();
 545             try {
 546                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 547                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 548
 549                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 550                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 551                                 fibUtil.getVpnInstanceOpData(rd);
 552                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 553                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 554                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 555                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 556                             }
 557                         }
 558                     }
 559                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 560                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 561                 }
 562             } finally {
 563                 lock.unlock();
 564             }
 565         });
 566         final List<InstructionInfo> instructions = new ArrayList<>();
 567         Uint64 subnetRouteMeta = Uint64.valueOf(BigInteger.valueOf(elanTag).shiftLeft(24)
 568             .or(BigInteger.valueOf(vpnId.longValue()).shiftLeft(1)));
 569         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 570         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 571         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 572                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 573         if (vrfEntry.getRoutePaths() != null) {
 574             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 575                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 576                     List<ActionInfo> actionsInfos = new ArrayList<>();
 577                     // reinitialize instructions list for LFIB Table
 578                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 579                     actionsInfos.add(new ActionPopMpls(etherType));
 580                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 581                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 582                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 583                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 584
 585                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions,
 586                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 587                 }
 588             }
 589         }
 590     }
 591
 592     private void installSubnetBroadcastAddrDropRule(final Uint64 dpnId, final String rd, final Uint32 vpnId,
 593             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 594         List<MatchInfo> matches = new ArrayList<>();
 595
 596         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 597                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 598         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 599         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 600         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 601
 602         InetAddress destPrefix;
 603         try {
 604             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 605         } catch (UnknownHostException e) {
 606             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 607                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 608             return;
 609         }
 610
 611         // Match on VpnId and SubnetBroadCast IP address
 612         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 613             MetaDataUtil.METADATA_MASK_VRFID));
 614         matches.add(MatchEthernetType.IPV4);
 615
 616         if (prefixLength != 0) {
 617             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 618         }
 619
 620         //Action is to drop the packet
 621         List<InstructionInfo> dropInstructions = new ArrayList<>();
 622         List<ActionInfo> actionsInfos = new ArrayList<>();
 623         actionsInfos.add(new ActionDrop());
 624         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 625
 626         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 627         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 628         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 629                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 630
 631         Flow flow = flowEntity.getFlowBuilder().build();
 632         String flowId = flowEntity.getFlowId();
 633         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 634         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 635
 636         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 637                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 638                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 639
 640         if (addOrRemove == NwConstants.ADD_FLOW) {
 641             tx.put(flowInstanceId,flow, true);
 642         } else {
 643             tx.delete(flowInstanceId);
 644         }
 645     }
 646
 647     /*
 648      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 649      * LportDispatcher table (via table 80)
 650      */
 651     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 652                                             final VrfEntry vrfEntry, int etherType) {
 653         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 654         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 655         // packet is commuted from Vpn2 to Vpn1.
 656         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 657         if (!interVpnLink.isActive()) {
 658             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 659                      interVpnLinkName, vrfEntry.getDestPrefix());
 660             return;
 661         }
 662
 663         Optional<Uint32> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 664         if (!optLportTag.isPresent()) {
 665             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 666             return;
 667         }
 668
 669         Long lportTag = optLportTag.get().toJava();
 670         Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 671         if (label == null) {
 672             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 673                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 674             return;
 675         }
 676         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 677         List<InstructionInfo> instructions = Arrays.asList(
 678             new InstructionApplyActions(actionsInfos),
 679             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 680                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 681                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 682                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 683             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 684         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 685         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 686
 687         for (Uint64 dpId : targetDpns) {
 688             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 689                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 690
 691             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 692                                /*writeTx*/null);
 693         }
 694     }
 695
 696
 697     /*
 698      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 699      */
 700     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 701                                                final VrfEntry vrfEntry, Uint32 vpnTag) {
 702         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 703         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 704             && vrfEntry.getRoutePaths().size() == 1);
 705         String destination = vrfEntry.getDestPrefix();
 706         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 707         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 708
 709         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 710         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 711         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 712         if (interVpnLink.getState().or(State.Error) != State.Active) {
 713             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 714                 destination, nextHop, interVpnLinkName);
 715             return;
 716         }
 717
 718         Optional<Uint32> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 719         if (!optOtherEndpointLportTag.isPresent()) {
 720             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 721                 vpnUuid, interVpnLinkName);
 722             return;
 723         }
 724
 725         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 726         if (targetDpns.isEmpty()) {
 727             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 728                 vpnUuid, interVpnLinkName);
 729             return;
 730         }
 731
 732         String[] values = destination.split("/");
 733         String destPrefixIpAddress = values[0];
 734         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 735
 736         List<MatchInfo> matches = new ArrayList<>();
 737         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag.longValue()),
 738             MetaDataUtil.METADATA_MASK_VRFID));
 739         matches.add(MatchEthernetType.IPV4);
 740
 741         if (prefixLength != 0) {
 742             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 743         }
 744
 745         List<Instruction> instructions =
 746             Arrays.asList(new InstructionWriteMetadata(
 747                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 748                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 749                             .L3VPN_SERVICE_INDEX)),
 750                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 751                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 752
 753         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 754         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 755         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 756             COOKIE_VM_FIB_TABLE, matches, instructions);
 757
 758         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 759             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 760
 761         for (Uint64 dpId : targetDpns) {
 762
 763             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 764                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 765                 dpId, interVpnLink.getInterVpnLinkName());
 766
 767             mdsalManager.installFlow(dpId, flowEntity);
 768         }
 769     }
 770
 771     private List<Uint64> createLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 772         List<Uint64> returnLocalDpnId = new ArrayList<>();
 773         String localNextHopIP = vrfEntry.getDestPrefix();
 774         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 775         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 776         if (localNextHopInfo == null) {
 777             boolean localNextHopSeen = false;
 778             List<Routes> vpnExtraRoutes = null;
 779             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 780             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 781             // FIXME: use an Identifier structure?
 782             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 783             lock.lock();
 784             try {
 785                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 786                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 787                         vpnName, usedRds, localNextHopIP);
 788                 if (LOG.isDebugEnabled()) {
 789                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 790                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 791                 }
 792
 793                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 794                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 795                     String ipPrefix;
 796                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 797                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 798                     } else {
 799                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 800                     }
 801                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 802                             ipPrefix);
 803                     if (localNextHopInfoLocal != null) {
 804                         localNextHopSeen = true;
 805                         Uint64 dpnId =
 806                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 807                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 808                         returnLocalDpnId.add(dpnId);
 809                     }
 810                 }
 811             } finally {
 812                 lock.unlock();
 813             }
 814             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 815                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 816                 if (optionalLabel.isPresent()) {
 817                     Uint32 label = optionalLabel.get();
 818                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 819                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 820                     final ReentrantLock labelLock = lockFor(lriKey);
 821                     labelLock.lock();
 822                     try {
 823                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 824                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 825                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 826                                     fibUtil.getVpnInstanceOpData(rd);
 827                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 828                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 829                                 if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains(
 830                                        vpnInstanceName)) {
 831                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 832                                     localNextHopIP = lri.getPrefix();
 833                                 } else {
 834                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 835                                     localNextHopIP = lri.getPrefix();
 836                                 }
 837                             }
 838                             if (localNextHopInfo != null) {
 839                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 840                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 841                                 if (vpnExtraRoutes.isEmpty()) {
 842                                     Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 843                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 844                                     returnLocalDpnId.add(dpnId);
 845                                 } else {
 846                                     for (Routes extraRoutes : vpnExtraRoutes) {
 847                                         Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 848                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 849                                         returnLocalDpnId.add(dpnId);
 850                                     }
 851                                 }
 852                             }
 853                         }
 854                     } finally {
 855                         labelLock.unlock();
 856                     }
 857                 }
 858             }
 859             if (returnLocalDpnId.isEmpty()) {
 860                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 861             }
 862         } else {
 863             Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 864                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 865             if (dpnId != null) {
 866                 returnLocalDpnId.add(dpnId);
 867             }
 868         }
 869         return returnLocalDpnId;
 870     }
 871
 872     private Uint64 checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 873                                                 final Uint32 vpnId, final String rd,
 874                                                 final VrfEntry vrfEntry,
 875                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 876                                                 int etherType) {
 877         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 878         if (localNextHopInfo != null) {
 879             long groupId;
 880             long localGroupId;
 881             final Uint64 dpnId = localNextHopInfo.getDpnId();
 882             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 883                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 884                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 885                 return dpnId;
 886             }
 887             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 888                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 889                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 890                 return dpnId;
 891             }
 892             if (!isVpnPresentInDpn(rd, dpnId)) {
 893                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 894                         vpnId, rd, dpnId.toString());
 895                 return Uint64.ZERO;
 896             }
 897             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 898             String prefix = vrfEntry.getDestPrefix();
 899             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 900             //The loadbalancing group is created only if the extra route has multiple nexthops
 901             //to avoid loadbalancing the discovered routes
 902             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 903                     && routes != null) {
 904                 if (vpnExtraRoutes.size() > 1) {
 905                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 906                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 907                 } else {
 908                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 909                     localGroupId = groupId;
 910                 }
 911             } else {
 912                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 913                         gwMacAddress);
 914                 localGroupId = groupId;
 915             }
 916             if (groupId == FibConstants.INVALID_GROUP_ID) {
 917                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 918                         prefix, rd, interfaceName, dpnId.toString());
 919                 return Uint64.ZERO;
 920             }
 921             final List<InstructionInfo> instructions = Collections.singletonList(
 922                     new InstructionApplyActions(
 923                             Collections.singletonList(new ActionGroup(groupId))));
 924             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 925                     new InstructionApplyActions(
 926                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 927             java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 928             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 929             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 930             jobCoordinator.enqueueJob(jobKey,
 931                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 932                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 933                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 934                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 935                         optLabel.ifPresent(label -> {
 936                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 937                                 LOG.debug(
 938                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 939                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 940                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 941                                         nextHopAddressList);
 942                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 943                                         NwConstants.ADD_FLOW, tx);
 944                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 945                             } else {
 946                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 947                                                 + "route. LFib and Terminating table entries will not be created.",
 948                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 949                             }
 950                         });
 951                     }
 952                 })));
 953             return dpnId;
 954         }
 955         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 956                 vpnName);
 957         return Uint64.ZERO;
 958     }
 959
 960     private boolean isVpnPresentInDpn(String rd, Uint64 dpnId)  {
 961         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 962         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 963         return dpnInVpn.isPresent();
 964     }
 965
 966     @Nullable
 967     private LabelRouteInfo getLabelRouteInfo(Uint32 label) {
 968         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 969     }
 970
 971     @Nullable
 972     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 973         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 974             .child(LabelRouteInfo.class, label).build();
 975         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 976         if (opResult.isPresent()) {
 977             return opResult.get();
 978         }
 979         return null;
 980     }
 981
 982     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 983             @Nullable TypedWriteTransaction<Operational> tx) {
 984         if (lri == null) {
 985             return true;
 986         }
 987
 988         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 989         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 990             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 991
 992         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 993             ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 994         if (vpnInstancesList.contains(vpnInstanceName)) {
 995             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 996             vpnInstancesList.remove(vpnInstanceName);
 997         }
 998         if (vpnInstancesList.isEmpty()) {
 999             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
1000             if (tx != null) {
1001                 tx.delete(lriId);
1002             } else {
1003                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
1004             }
1005             return true;
1006         } else {
1007             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
1008             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
1009             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1010         }
1011         return false;
1012     }
1013
1014     void makeTunnelTableEntry(Uint64 dpId, Uint32 label, long groupId/*String egressInterfaceName*/,
1015                                       TypedWriteTransaction<Configuration> tx) {
1016         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1017
1018         createTerminatingServiceActions(dpId, label, actionsInfos, tx);
1019
1020         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1021             dpId, label, groupId);
1022     }
1023
1024     public void createTerminatingServiceActions(Uint64 destDpId, Uint32 label, List<ActionInfo> actionsInfos,
1025                                                 TypedWriteTransaction<Configuration> tx) {
1026         List<MatchInfo> mkMatches = new ArrayList<>();
1027
1028         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1029             destDpId, label, actionsInfos);
1030
1031         // Matching metadata
1032         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1033         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1034
1035         List<InstructionInfo> mkInstructions = new ArrayList<>();
1036         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1037
1038         FlowEntity terminatingServiceTableFlowEntity =
1039             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1040             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1041                     FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1042                     String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1043                     Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()),
1044                     mkMatches, mkInstructions);
1045
1046         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1047
1048         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1049
1050         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1051         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1052             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1053             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1054             .child(Flow.class, flowKey).build();
1055         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1056     }
1057
1058     private void removeTunnelTableEntry(Uint64 dpId, Uint32 label, TypedWriteTransaction<Configuration> tx) {
1059         FlowEntity flowEntity;
1060         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1061         List<MatchInfo> mkMatches = new ArrayList<>();
1062         // Matching metadata
1063         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1064         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1065             NwConstants.INTERNAL_TUNNEL_TABLE,
1066             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1067                 FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1068                 String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1069                 Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()), mkMatches, null);
1070         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1071         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1072         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1073             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1074             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1075
1076         tx.delete(flowInstanceId);
1077         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1078     }
1079
1080     public List<Uint64> deleteLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry) {
1081         List<Uint64> returnLocalDpnId = new ArrayList<>();
1082         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1083         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1084         boolean shouldUpdateNonEcmpLocalNextHop = true;
1085         if (localNextHopInfo == null) {
1086             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1087             if (usedRds.size() > 1) {
1088                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1089                         vrfEntry.getDestPrefix(), vpnName, rd);
1090                 return returnLocalDpnId;
1091             }
1092             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1093             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1094             //in the vpn
1095             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1096                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1097             if (extraRouteOptional.isPresent()) {
1098                 Routes extraRoute = extraRouteOptional.get();
1099                 String ipPrefix;
1100                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1101                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1102                 } else {
1103                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1104                 }
1105                 if (extraRoute.getNexthopIpList().size() > 1) {
1106                     shouldUpdateNonEcmpLocalNextHop = false;
1107                 }
1108                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1109                 if (localNextHopInfo != null) {
1110                     String localNextHopIP = localNextHopInfo.getIpAddress();
1111                     Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1112                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1113                     if (!dpnId.equals(Uint64.ZERO)) {
1114                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1115                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1116                         returnLocalDpnId.add(dpnId);
1117                     }
1118                 } else {
1119                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1120                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1121                 }
1122             }
1123
1124             if (localNextHopInfo == null) {
1125                 /* Imported VRF entry */
1126                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1127                 if (optionalLabel.isPresent()) {
1128                     Uint32 label = optionalLabel.get();
1129                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1130                     LabelRouteInfo lri = getLabelRouteInfo(label);
1131                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1132                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1133                         prefixBuilder.setDpnId(lri.getDpnId());
1134                         Uint64 dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1135                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1136                         if (!dpnId.equals(Uint64.ZERO)) {
1137                             returnLocalDpnId.add(dpnId);
1138                         }
1139                     }
1140                 }
1141             }
1142
1143         } else {
1144             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1145             String localNextHopIP = localNextHopInfo.getIpAddress();
1146             Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1147                     shouldUpdateNonEcmpLocalNextHop);
1148             if (!dpnId.equals(Uint64.ZERO)) {
1149                 returnLocalDpnId.add(dpnId);
1150             }
1151         }
1152
1153         return returnLocalDpnId;
1154     }
1155
1156     private Uint64 checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1157             final String vpnName, final Uint32 vpnId, final String rd, final VrfEntry vrfEntry,
1158             boolean shouldUpdateNonEcmpLocalNextHop) {
1159         if (localNextHopInfo != null) {
1160             final Uint64 dpnId = localNextHopInfo.getDpnId();
1161             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1162                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1163                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1164                 return dpnId;
1165             }
1166             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1167                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1168                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1169                 return dpnId;
1170             }
1171
1172             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1173                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1174                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1175                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1176                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1177                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1178                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1179                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1180                                         NwConstants.DEL_FLOW, tx);
1181                                 removeTunnelTableEntry(dpnId, label, tx);
1182                             });
1183                         }
1184                     }
1185                 })));
1186             //TODO: verify below adjacency call need to be optimized (?)
1187             //In case of the removal of the extra route, the loadbalancing group is updated
1188             if (shouldUpdateNonEcmpLocalNextHop) {
1189                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1190             }
1191             return dpnId;
1192         }
1193         return Uint64.ZERO;
1194     }
1195
1196     private void createRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1197             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1198         if (tx == null) {
1199             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1200                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1201                 "Error creating remote FIB entry");
1202             return;
1203         }
1204
1205         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1206         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", vrfEntry.getDestPrefix(), rd,
1207                 remoteDpnId);
1208
1209         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1210             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1211             return;
1212         }
1213         // Handling static VRF entries
1214         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1215         List<Routes> vpnExtraRoutes =
1216                 VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, vpnName, usedRds, vrfEntry.getDestPrefix());
1217         if (!vpnExtraRoutes.isEmpty()) {
1218             programRemoteFibWithLoadBalancingGroups(remoteDpnId, vpnId, rd, vrfEntry, vpnExtraRoutes);
1219         } else {
1220             // Program in case of other static VRF entries like floating IPs
1221             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1222         }
1223     }
1224
1225     private void programRemoteFibWithLoadBalancingGroups(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1226             final VrfEntry vrfEntry, List<Routes> vpnExtraRoutes) {
1227         // create loadbalancing groups for extra routes only when the extra route is
1228         // present behind multiple VMs
1229         // Obtain the local routes for this particular dpn.
1230         java.util.Optional<Routes> routes = vpnExtraRoutes.stream().filter(route -> {
1231             Prefixes prefixToInterface =
1232                     fibUtil.getPrefixToInterface(vpnId, FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1233             if (prefixToInterface == null) {
1234                 return false;
1235             }
1236             return remoteDpnId.equals(prefixToInterface.getDpnId());
1237         }).findFirst();
1238         long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1239                 routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1240         if (groupId == FibConstants.INVALID_GROUP_ID) {
1241             LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", vrfEntry.getDestPrefix(), rd,
1242                     remoteDpnId);
1243             return;
1244         }
1245         List<ActionInfo> actionInfos = Collections.singletonList(new ActionGroup(groupId));
1246         List<InstructionInfo> instructions = Lists.newArrayList(new InstructionApplyActions(actionInfos));
1247         String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1248         jobCoordinator.enqueueJob(jobKey,
1249             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1250                 baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1251                         NwConstants.ADD_FLOW, txn, null);
1252             })));
1253
1254         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1255     }
1256
1257     private void programRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1258             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1259         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1260         if (adjacencyResults.isEmpty()) {
1261             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", vrfEntry.getRoutePaths(), rd,
1262                     remoteDpnId);
1263             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1264             return;
1265         }
1266         baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, TransactionAdapter.toWriteTransaction(tx),
1267                 rd, adjacencyResults, null);
1268         LOG.debug("Successfully programmed FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1269     }
1270
1271     protected void cleanUpOpDataForFib(Uint32 vpnId, String primaryRd, final VrfEntry vrfEntry) {
1272     /* Get interface info from prefix to interface mapping;
1273         Use the interface info to get the corresponding vpn interface op DS entry,
1274         remove the adjacency corresponding to this fib entry.
1275         If adjacency removed is the last adjacency, clean up the following:
1276          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1277          - prefix to interface entry
1278          - vpn interface op DS
1279      */
1280         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1281         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1282         if (prefixInfo == null) {
1283             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1284             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1285             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1286             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
1287                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1288                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1289                     if (nextHopIp != null) {
1290                         String ipPrefix;
1291                         if (isIpv4Address(nextHopIp)) {
1292                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1293                         } else {
1294                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1295                         }
1296                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1297                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1298                     }
1299                 }
1300             }
1301             if (prefixInfo == null) {
1302                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1303                 if (optionalLabel.isPresent()) {
1304                     Uint32 label = optionalLabel.get();
1305                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1306                     LabelRouteInfo lri = getLabelRouteInfo(label);
1307                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1308                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1309                         prefixBuilder.setDpnId(lri.getDpnId());
1310                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1311                         prefixBuilder.setIpAddress(lri.getPrefix());
1312                         prefixInfo = prefixBuilder.build();
1313                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1314                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1315                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1316                     }
1317                 }
1318             }
1319         } else {
1320             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1321         }
1322     }
1323
1324     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1325                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1326
1327         if (prefixInfo == null) {
1328             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1329                     vrfEntry.getDestPrefix(), vpnId, rd);
1330             return; //Don't have any info for this prefix (shouldn't happen); need to return
1331         }
1332
1333         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1334             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1335                     vrfEntry.getDestPrefix(), vpnId, rd);
1336             return;
1337         }
1338
1339         String ifName = prefixInfo.getVpnInterfaceName();
1340         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1341             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1342     }
1343
1344     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1345         Prefixes prefixInfo;
1346         Uint32 vpnId;
1347         String rd;
1348         VrfEntry vrfEntry;
1349         Routes extraRoute;
1350
1351         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1352                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1353             this.prefixInfo = prefixInfo;
1354             this.vpnId = vpnId;
1355             this.rd = rd;
1356             this.vrfEntry = vrfEntry;
1357             this.extraRoute = extraRoute;
1358         }
1359
1360         @Override
1361         public List<ListenableFuture<Void>> call() {
1362             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1363             // to call the respective helpers.
1364             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1365                 //First Cleanup LabelRouteInfo
1366                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1367                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1368                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1369                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1370                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1371                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1372                         final ReentrantLock lock = lockFor(lriKey);
1373                         lock.lock();
1374                         try {
1375                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1376                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1377                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1378                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1379                                         fibUtil.getVpnInstanceOpData(rd);
1380                                 String vpnInstanceName = "";
1381                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1382                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1383                                 }
1384                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1385                                 if (lriRemoved) {
1386                                     String parentRd = lri.getParentVpnRd();
1387                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1388                                             parentRd, vrfEntry.getDestPrefix()));
1389                                 }
1390                             } else {
1391                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1392                                         rd, vrfEntry.getDestPrefix()));
1393                             }
1394                         } finally {
1395                             lock.unlock();
1396                         }
1397                     });
1398                 }
1399                 String ifName = prefixInfo.getVpnInterfaceName();
1400                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1401                 String vpnName = null;
1402
1403                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1404                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1405                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1406                     if (vpnNameOpt.isPresent()) {
1407                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1408                     }
1409                 }
1410                 if (optVpnName.isPresent()) {
1411                     vpnName = optVpnName.get();
1412                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1413                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1414                     if (opVpnInterface.isPresent()) {
1415                         Uint32 associatedVpnId = fibUtil.getVpnId(vpnName);
1416                         if (Objects.equals(vpnId, associatedVpnId)) {
1417                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1418                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1419                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1420                             return;
1421                         } else {
1422                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1423                                     vrfEntry.getDestPrefix(), associatedVpnId);
1424                         }
1425                     }
1426                 }
1427                 if (extraRoute != null) {
1428                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1429                     //Only one used Rd present in case of removal event
1430                     String usedRd = usedRds.get(0);
1431                     if (optVpnName.isPresent()) {
1432                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1433                                         vrfEntry.getDestPrefix()));
1434                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1435                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1436                     }
1437                 }
1438                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1439             }));
1440         }
1441     }
1442
1443     /**
1444      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1445      * Remove Adjacency from VPNInterfaceOpData.
1446      * if Adjacency != primary.
1447      * if Adjacency == primary , then mark it for deletion.
1448      * Remove entire VPNinterfaceOpData Entry.
1449      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1450      * @param vrfEntry - VrfEntry removed
1451      * @param ifName - Interface name from VRFentry
1452      * @param vpnName - VPN name of corresponding VRF
1453      * @param tx - ReadWrite Tx
1454      */
1455     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
1456             justification = "https://github.com/spotbugs/spotbugs/issues/811")
1457     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1458                                                           TypedReadWriteTransaction<Operational> tx)
1459             throws ExecutionException, InterruptedException {
1460         InstanceIdentifier<Adjacency> adjacencyIid =
1461                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1462         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1463         if (adjacencyOptional.isPresent()) {
1464             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1465                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1466             } else {
1467                 tx.merge(adjacencyIid,
1468                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1469             }
1470         }
1471
1472         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1473
1474         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1475             return;
1476         }
1477
1478         @NonNull List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
1479         if (adjacencies.size() <= 2
1480                 && adjacencies.stream().allMatch(adjacency ->
1481                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1482                         && adjacency.isMarkedForDeletion() != null
1483                         && adjacency.isMarkedForDeletion()
1484         )) {
1485             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1486             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1487         }
1488     }
1489
1490     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1491         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1492         final String rd = vrfTableKey.getRouteDistinguisher();
1493         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1494         if (vpnInstance == null) {
1495             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1496             return;
1497         }
1498         final Collection<VpnToDpnList> vpnToDpnList;
1499         if (vrfEntry.getParentVpnRd() != null
1500                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1501             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1502             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1503             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1504                     vpnInstance.getVpnToDpnList();
1505             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1506                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1507         } else {
1508             vpnToDpnList = vpnInstance.getVpnToDpnList();
1509         }
1510
1511         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1512         final java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1513         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1514         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1515         if (subnetRoute != null) {
1516             long elanTag = subnetRoute.getElantag().toJava();
1517             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1518                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1519             if (vpnToDpnList != null) {
1520                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1521                     () -> Collections.singletonList(
1522                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1523                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1524
1525                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(),
1526                                     vpnInstance.getVpnId(),
1527                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1528                                     TransactionAdapter.toWriteTransaction(tx), null);
1529                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1530                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(),
1531                                         label, null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1532                                 }
1533
1534                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
1535                                     vpnInstance.getVpnId(),
1536                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1537                             }
1538                         })));
1539             }
1540             optionalLabel.ifPresent(label -> {
1541                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1542                 final ReentrantLock lock = lockFor(lriKey);
1543                 lock.lock();
1544                 try {
1545                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1546                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1547                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1548                                 fibUtil.getVpnInstanceOpData(rd);
1549                         String vpnInstanceName = "";
1550                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1551                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1552                         }
1553                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1554                         if (lriRemoved) {
1555                             String parentRd = lri.getParentVpnRd();
1556                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1557                                     parentRd, vrfEntry.getDestPrefix()));
1558                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1559                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1560                         }
1561                     } else {
1562                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1563                                 rd, vrfEntry.getDestPrefix()));
1564                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1565                                 label, rd, vrfEntry.getDestPrefix());
1566                     }
1567                 } finally {
1568                     lock.unlock();
1569                 }
1570             });
1571             return;
1572         }
1573
1574         final List<Uint64> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1575             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1576         if (vpnToDpnList != null) {
1577             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1578                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1579             String jobKey;
1580             Optional<Routes> extraRouteOptional;
1581             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1582             if (usedRds != null && !usedRds.isEmpty()) {
1583                 if (usedRds.size() > 1) {
1584                     LOG.error("The extra route prefix is still present in some DPNs");
1585                     return ;
1586                 } else {
1587                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1588                     //is not present in any other DPN
1589                     extraRouteOptional = VpnExtraRouteHelper
1590                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1591                 }
1592             } else {
1593                 extraRouteOptional = Optional.absent();
1594             }
1595
1596             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1597                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1598                     if (localDpnIdList.size() <= 0) {
1599                         for (VpnToDpnList curDpn : vpnToDpnList) {
1600                             baseVrfEntryHandler.deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(),
1601                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1602                                 TransactionAdapter.toWriteTransaction(tx));
1603                         }
1604                     } else {
1605                         for (Uint64 localDpnId : localDpnIdList) {
1606                             for (VpnToDpnList curDpn : vpnToDpnList) {
1607                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1608                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1609                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1610                                         TransactionAdapter.toWriteTransaction(tx));
1611                                 }
1612                             }
1613                         }
1614                     }
1615                     if (extraRouteOptional.isPresent()) {
1616                         //Remove select groups only for extra-routes
1617                         nextHopManager.removeNextHopPointer(nextHopManager
1618                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1619                         nextHopManager.removeNextHopPointer(nextHopManager
1620                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1621                     }
1622                 })), MAX_RETRIES);
1623         }
1624
1625         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1626         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1627         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1628
1629         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1630         // of the interVpnLink.
1631         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1632         if (optVpnUuid.isPresent()) {
1633             String vpnUuid = optVpnUuid.get();
1634             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1635                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1636                 if (optInterVpnLink.isPresent()) {
1637                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1638                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1639                         // This is route that points to the other endpoint of an InterVpnLink
1640                         // In that case, we should look for the FIB table pointing to
1641                         // LPortDispatcher table and remove it.
1642                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1643                     }
1644                 }
1645             });
1646         }
1647
1648     }
1649
1650     private void makeLFibTableEntry(Uint64 dpId, Uint32 label, @Nullable List<InstructionInfo> instructions,
1651                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1652         if (tx == null) {
1653             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1654                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1655                 "Error making LFIB table entry");
1656             return;
1657         }
1658
1659         List<MatchInfo> matches = new ArrayList<>();
1660         matches.add(MatchEthernetType.MPLS_UNICAST);
1661         matches.add(new MatchMplsLabel(label.longValue()));
1662
1663         // Install the flow entry in L3_LFIB_TABLE
1664         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1665
1666         FlowEntity flowEntity;
1667         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1668             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1669         Flow flow = flowEntity.getFlowBuilder().build();
1670         String flowId = flowEntity.getFlowId();
1671         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1672         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1673         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1674             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1675             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1676
1677         if (addOrRemove == NwConstants.ADD_FLOW) {
1678             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1679         } else {
1680             tx.delete(flowInstanceId);
1681         }
1682
1683         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1684             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1685     }
1686
1687     public void populateFibOnNewDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1688                                     final FutureCallback<List<Void>> callback) {
1689         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1690         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1691             () -> {
1692                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1693                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1694                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1695                         LogicalDatastoreType.CONFIGURATION, id);
1696                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1697                 if (!vrfTable.isPresent()) {
1698                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1699                     if (callback != null) {
1700                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1701                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1702                     }
1703                     return futures;
1704                 }
1705
1706                 final ReentrantLock lock = lockFor(vpnInstance);
1707                 lock.lock();
1708                 try {
1709                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1710                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1711                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1712                             if (subnetRoute != null) {
1713                                 long elanTag = subnetRoute.getElantag().toJava();
1714                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1715                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1716                                         tx);
1717                                 continue;
1718                             }
1719                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1720                             if (routerInt != null) {
1721                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1722                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1723                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1724                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1725                                         NwConstants.ADD_FLOW);
1726                                 continue;
1727                             }
1728                             //Handle local flow creation for imports
1729                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1730                                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1731                                 if (optionalLabel.isPresent()) {
1732                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1733                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1734                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1735                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1736                                             try {
1737                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1738                                                         vrfEntry.getDestPrefix());
1739                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1740                                             } catch (IllegalArgumentException ex) {
1741                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1742                                                         vrfEntry.getDestPrefix());
1743                                             }
1744                                             continue;
1745                                         }
1746                                     }
1747                                 }
1748                             }
1749                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1750                                     vrfEntry, dpnId);
1751                             if (shouldCreateRemoteFibEntry) {
1752                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1753                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1754                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1755                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1756                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1757                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1758                                 } else {
1759                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1760                                             vrfEntry, tx);
1761                                 }
1762                             }
1763                         }
1764                     }));
1765                     if (callback != null) {
1766                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1767                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1768                     }
1769                 } finally {
1770                     lock.unlock();
1771                 }
1772                 return futures;
1773             });
1774     }
1775
1776     public void populateExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1777                                             final String localNextHopIp, final String remoteNextHopIp) {
1778         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1779             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1780         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1781         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1782         List<SubTransaction> txnObjects =  new ArrayList<>();
1783         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1784         if (vrfTable.isPresent()) {
1785             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1786                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1787                     final ReentrantLock lock = lockFor(vpnInstance);
1788                     lock.lock();
1789                     try {
1790                         vrfTable.get().nonnullVrfEntry().stream()
1791                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1792                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1793                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1794                     } finally {
1795                         lock.unlock();
1796                     }
1797                 })));
1798         }
1799     }
1800
1801     public void manageRemoteRouteOnDPN(final boolean action,
1802                                        final Uint64 localDpnId,
1803                                        final Uint32 vpnId,
1804                                        final String rd,
1805                                        final String destPrefix,
1806                                        final String destTepIp,
1807                                        final Uint32 label) {
1808         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1809
1810         if (vpnInstance == null) {
1811             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1812             return;
1813         }
1814
1815         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1816             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1817                 final ReentrantLock lock = lockFor(vpnInstance);
1818                 lock.lock();
1819                 try {
1820                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1821                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1822                     if (vrfEntry == null) {
1823                         return;
1824                     }
1825                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1826                             action, localDpnId, vpnId, rd, destPrefix);
1827                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1828                     VrfEntry modVrfEntry;
1829                     if (routePathList == null || routePathList.isEmpty()) {
1830                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1831                                 Collections.singletonList(destTepIp),
1832                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1833                     } else {
1834                         modVrfEntry = vrfEntry;
1835                     }
1836
1837                     if (action) {
1838                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1839                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1840                                 modVrfEntry, tx);
1841                     } else {
1842                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1843                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1844                                 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1845                         if (usedRds.size() > 1) {
1846                             LOG.debug("The extra route prefix is still present in some DPNs");
1847                             return;
1848                         }
1849                         //Is this fib route an extra route? If yes, get the nexthop which would be
1850                         //an adjacency in the vpn
1851                         Optional<Routes> extraRouteOptional = Optional.absent();
1852                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC && usedRds.size() != 0) {
1853                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1854                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1855                                     usedRds.get(0), vrfEntry.getDestPrefix());
1856                         }
1857                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1858                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1859                     }
1860                 } finally {
1861                     lock.unlock();
1862                 }
1863             })));
1864     }
1865
1866     public void cleanUpDpnForVpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1867                                  final FutureCallback<List<Void>> callback) {
1868         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1869         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1870             () -> {
1871                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1872                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1873                 List<SubTransaction> txnObjects =  new ArrayList<>();
1874                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1875                         LogicalDatastoreType.CONFIGURATION, id);
1876                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1877                 if (vrfTable.isPresent()) {
1878                     final ReentrantLock lock = lockFor(vpnInstance);
1879                     lock.lock();
1880                     try {
1881                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1882                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1883                             for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1884                                 /* parentRd is only filled for external PNF cases where the interface on the external
1885                                  * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1886                                  * #fibUtil.isInterfacePresentInDpn().
1887                                 * */
1888                                 String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1889                                         : rd;
1890                                 /* Handle subnet routes here */
1891                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1892                                 if (subnetRoute != null && !fibUtil
1893                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1894                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1895                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1896                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1897                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1898                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1899                                     if (routePaths != null) {
1900                                         for (RoutePaths routePath : routePaths) {
1901                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1902                                                     DEFAULT_FIB_FLOW_PRIORITY,
1903                                                     NwConstants.DEL_FLOW, tx);
1904                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1905                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1906                                                     vrfEntry.getDestPrefix());
1907                                         }
1908                                     }
1909                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1910                                             NwConstants.DEL_FLOW, tx);
1911                                     continue;
1912                                 }
1913                                 // ping responder for router interfaces
1914                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1915                                 if (routerInt != null) {
1916                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1917                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1918                                             routerInt.getMacAddress());
1919                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1920                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1921                                             NwConstants.DEL_FLOW);
1922                                     continue;
1923                                 }
1924
1925                                 //Handle local flow deletion for imports
1926                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1927                                     java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1928                                     if (optionalLabel.isPresent()) {
1929                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1930                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1931                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1932                                                 lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1933                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1934                                         }
1935                                     }
1936                                 }
1937
1938                                 // Passing null as we don't know the dpn
1939                                 // to which prefix is attached at this point
1940                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1941                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1942                                 Optional<Routes> extraRouteOptional;
1943                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1944                                 //an adjacency in the vpn
1945                                 if (usedRds != null && !usedRds.isEmpty()) {
1946                                     if (usedRds.size() > 1) {
1947                                         LOG.error("The extra route prefix is still present in some DPNs");
1948                                         return;
1949                                     } else {
1950                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1951                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1952
1953                                     }
1954                                 } else {
1955                                     extraRouteOptional = Optional.absent();
1956                                 }
1957                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1958                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1959                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1960                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1961                                 } else {
1962                                     if (subnetRoute == null || !fibUtil
1963                                             .isInterfacePresentInDpn(parentRd, dpnId)) {
1964                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1965                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1966                                             TransactionAdapter.toWriteTransaction(tx));
1967                                     }
1968                                 }
1969                             }
1970                         }));
1971                     } finally {
1972                         lock.unlock();
1973                     }
1974                     if (callback != null) {
1975                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1976                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1977                     }
1978                 } else {
1979                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1980                 }
1981                 return futures;
1982             });
1983
1984     }
1985
1986     public void cleanUpExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1987                                            final String localNextHopIp, final String remoteNextHopIp) {
1988         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1989                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1990             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1991         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1992         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1993         List<SubTransaction> txnObjects =  new ArrayList<>();
1994         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1995         if (vrfTable.isPresent()) {
1996             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1997                 () -> {
1998                     final ReentrantLock lock = lockFor(vpnInstance);
1999                     lock.lock();
2000                     try {
2001                         return Collections.singletonList(
2002                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
2003                                 tx -> vrfTable.get().nonnullVrfEntry().stream()
2004                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2005                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
2006                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
2007                                         txnObjects))));
2008                     } finally {
2009                         lock.unlock();
2010                     }
2011                 });
2012         }
2013     }
2014
2015     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2016         InstanceIdentifierBuilder<VrfTables> idBuilder =
2017             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2018         return idBuilder.build();
2019     }
2020
2021     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2022         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2023                 .FLOWID_SEPARATOR + nextHop;
2024     }
2025
2026     private String getTableMissFlowRef(Uint64 dpnId, short tableId, Uint32 tableMiss) {
2027         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2028                 + tableMiss + FLOWID_PREFIX;
2029     }
2030
2031     @Nullable
2032     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2033         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2034             .child(VrfTables.class, new VrfTablesKey(rd))
2035             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2036         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2037         if (vrfEntry.isPresent()) {
2038             return vrfEntry.get();
2039         }
2040         return null;
2041     }
2042
2043     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2044                                              final String vpnName,
2045                                              final VrfEntry vrfEntry) {
2046         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2047
2048         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2049         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2050
2051         if (targetDpns.isEmpty()) {
2052             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2053             return;
2054         }
2055
2056         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2057         java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2058
2059         // delete from FIB
2060         //
2061         optNextHop.ifPresent(nextHop -> {
2062             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2063             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2064             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2065                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2066
2067             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2068             for (Uint64 dpId : targetDpns) {
2069                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2070                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2071
2072                 mdsalManager.removeFlow(dpId, flow);
2073             }
2074         });
2075
2076         // delete from LFIB
2077         //
2078         optLabel.ifPresent(label -> {
2079             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2080
2081             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2082                 for (Uint64 dpId : targetDpns) {
2083                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2084                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2085                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY,
2086                             NwConstants.DEL_FLOW, tx);
2087                 }
2088             }), LOG, "Error removing flows");
2089         });
2090     }
2091
2092     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2093             List<String> nextHopAddressList, LabelRouteInfo lri) {
2094         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2095                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2096     }
2097
2098     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Uint32 vpnId, VrfEntry vrfEntry, Uint64 dpnId) {
2099         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2100             return true;
2101         }
2102
2103         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2104         if (prefix != null) {
2105             Uint64 prefixDpnId = prefix.getDpnId();
2106             if (dpnId.equals(prefixDpnId)) {
2107                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2108                         vrfEntry, dpnId);
2109                 return false;
2110             }
2111         }
2112         return true;
2113     }
2114
2115     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2116         // FIXME: use vpnInstance.key() instead?
2117         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2118     }
2119
2120     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2121         return JvmGlobalLocks.getLockFor(label);
2122     }
2123 }