fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14 import static org.opendaylight.netvirt.fibmanager.FibUtil.nullToEmpty;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.net.InetAddress;
  25 import java.net.UnknownHostException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.List;
  31 import java.util.Objects;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import javax.annotation.Nullable;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  43 import org.opendaylight.genius.infra.Datastore.Configuration;
  44 import org.opendaylight.genius.infra.Datastore.Operational;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.TransactionAdapter;
  49 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  50 import org.opendaylight.genius.infra.TypedWriteTransaction;
  51 import org.opendaylight.genius.mdsalutil.ActionInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  54 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  55 import org.opendaylight.genius.mdsalutil.MatchInfo;
  56 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  57 import org.opendaylight.genius.mdsalutil.NWUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  65 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.ServiceIndex;
  72 import org.opendaylight.genius.utils.batching.SubTransaction;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  75 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  76 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  77 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  78 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  79 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  80 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  81 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  82 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  83 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  84 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 117 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 118 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 119 import org.slf4j.Logger;
 120 import org.slf4j.LoggerFactory;
 121
 122
 123 @Singleton
 124 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 125
 126     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 127     private static final String FLOWID_PREFIX = "L3.";
 128     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 129     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 130     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 131     private static final int LFIB_INTERVPN_PRIORITY = 15;
 132     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 133     private static final int MAX_RETRIES = 3;
 134     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 135
 136     private final DataBroker dataBroker;
 137     private final ManagedNewTransactionRunner txRunner;
 138     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 139     private final IMdsalApiManager mdsalManager;
 140     private final NexthopManager nextHopManager;
 141     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 142     private final BaseVrfEntryHandler baseVrfEntryHandler;
 143     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final IElanService elanManager;
 146     private final FibUtil fibUtil;
 147     private final InterVpnLinkCache interVpnLinkCache;
 148     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 149     private final UpgradeState upgradeState;
 150     private final DataTreeEventCallbackRegistrar eventCallbacks;
 151
 152     @Inject
 153     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 154                             final NexthopManager nexthopManager,
 155                             final IElanService elanManager,
 156                             final BaseVrfEntryHandler vrfEntryHandler,
 157                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 158                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 159                             final JobCoordinator jobCoordinator,
 160                             final FibUtil fibUtil,
 161                             final InterVpnLinkCache interVpnLinkCache,
 162                             final UpgradeState upgradeState,
 163                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 164         super(VrfEntry.class, VrfEntryListener.class);
 165         this.dataBroker = dataBroker;
 166         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 167         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 168         this.mdsalManager = mdsalApiManager;
 169         this.nextHopManager = nexthopManager;
 170         this.elanManager = elanManager;
 171         this.baseVrfEntryHandler = vrfEntryHandler;
 172         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 173         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 174         this.jobCoordinator = jobCoordinator;
 175         this.fibUtil = fibUtil;
 176         this.interVpnLinkCache = interVpnLinkCache;
 177         this.upgradeState = upgradeState;
 178         this.eventCallbacks = eventCallbacks;
 179     }
 180
 181     @Override
 182     @PostConstruct
 183     public void init() {
 184         LOG.info("{} init", getClass().getSimpleName());
 185         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 186     }
 187
 188     @Override
 189     @SuppressWarnings("checkstyle:IllegalCatch")
 190     public void close() {
 191         closeables.forEach(c -> {
 192             try {
 193                 c.close();
 194             } catch (Exception e) {
 195                 LOG.warn("Error closing {}", c, e);
 196             }
 197         });
 198     }
 199
 200     @Override
 201     protected VrfEntryListener getDataTreeChangeListener() {
 202         return VrfEntryListener.this;
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 207         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 208     }
 209
 210     @Override
 211     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 212         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 213         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 214         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 215                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 216         addFibEntries(identifier, vrfEntry, rd);
 217         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 218                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 219     }
 220
 221     //This method is temporary. Eventually Factory design pattern will be used to get
 222     // right VrfEntryhandle and invoke its methods.
 223     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 224         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 225             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 226             return;
 227         }
 228         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 229             LOG.info("EVPN flows need to be programmed.");
 230             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 231                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 232             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 233             closeables.add(evpnVrfEntryHandler);
 234             return;
 235         }
 236         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 237         if (routerInt != null) {
 238             // ping responder for router interfaces
 239             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 240             return;
 241         }
 242         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 243             createFibEntries(identifier, vrfEntry);
 244             return;
 245         }
 246     }
 247
 248     @Override
 249     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 250         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 251         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 252         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 253                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 254         removeFibEntries(identifier, vrfEntry, rd);
 255         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 256             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 257     }
 258
 259     //This method is temporary. Eventually Factory design pattern will be used to get
 260     // right VrfEntryhandle and invoke its methods.
 261     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 262         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 263             LOG.info("EVPN flows to be deleted");
 264             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 265                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 266             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 267             closeables.add(evpnVrfEntryHandler);
 268             return;
 269         }
 270         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 271         if (routerInt != null) {
 272             // ping responder for router interfaces
 273             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 274             return;
 275         }
 276         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 277             deleteFibEntries(identifier, vrfEntry);
 278             return;
 279         }
 280         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 281             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 282             return;
 283         }
 284     }
 285
 286     @Override
 287     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 288     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 289     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 290     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 291         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 292         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 293         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 294                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 295         // Handle BGP Routes first
 296         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 297             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 298             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 299                     rd, update.getDestPrefix(), update.getRoutePaths());
 300             return;
 301         }
 302
 303         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 304             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 305             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 306             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 307
 308             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 309             //updated vrf entry route path is null. This can happen during tunnel events.
 310             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 311             List<String> usedRds = new ArrayList<>();
 312             if (optVpnInstance.isPresent()) {
 313                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 314                         update.getDestPrefix());
 315             }
 316             // If original VRF Entry had nexthop null , but update VRF Entry
 317             // has nexthop , route needs to be created on remote Dpns
 318             if (originalRoutePath == null || originalRoutePath.isEmpty()
 319                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 320                 // TODO(vivek): Though ugly, Not handling this code now, as each
 321                 // tep add event will invoke flow addition
 322                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 323                         + " This event is IGNORED here.", update.getDestPrefix());
 324                 return;
 325             }
 326
 327             // If original VRF Entry had valid nexthop , but update VRF Entry
 328             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 329             if (updateRoutePath == null || updateRoutePath.isEmpty()
 330                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 331                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 332                         + "This event is IGNORED here.", update.getDestPrefix());
 333                 return;
 334             }
 335             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 336             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 337             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 338             List<ListenableFuture<Void>> futures = new ArrayList<>();
 339             ListenableFuture<Void> configFuture =
 340                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 341                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 342                         nextHopsRemoved.parallelStream()
 343                             .forEach(nextHopRemoved -> {
 344                                 try {
 345                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 346                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 347                                 } catch (ExecutionException | InterruptedException e) {
 348                                     throw new RuntimeException(e);
 349                                 }
 350                             }))));
 351             futures.add(configFuture);
 352             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 353                 @Override
 354                 public void onSuccess(Void result) {
 355                     createFibEntries(identifier, update);
 356                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 357                             rd, update.getDestPrefix(), update.getRoutePaths());
 358                 }
 359
 360                 @Override
 361                 public void onFailure(Throwable throwable) {
 362                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 363                             update, throwable);
 364                 }
 365             }, MoreExecutors.directExecutor());
 366             return;
 367         }
 368
 369         //Handle all other routes only on a cluster reboot
 370         if (original.equals(update)) {
 371             //Reboot use-case
 372             createFibEntries(identifier, update);
 373             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 374                     rd, update.getDestPrefix(), update.getRoutePaths());
 375             return;
 376         }
 377
 378         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 379                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 380     }
 381
 382     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 383         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 384         List<SubTransaction> txnObjects =  new ArrayList<>();
 385         final VpnInstanceOpDataEntry vpnInstance =
 386                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 387         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 388         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 389                 + " has null vpnId!");
 390         final Collection<VpnToDpnList> vpnToDpnList;
 391         if (vrfEntry.getParentVpnRd() != null
 392                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 393             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 394             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 395             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 396                 vpnInstance.getVpnToDpnList();
 397             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 398                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 399         } else {
 400             vpnToDpnList = vpnInstance.getVpnToDpnList();
 401         }
 402         final Long vpnId = vpnInstance.getVpnId();
 403         final String rd = vrfTableKey.getRouteDistinguisher();
 404         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 405         if (subnetRoute != null) {
 406             final long elanTag = subnetRoute.getElantag();
 407             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 408                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 409             if (vpnToDpnList != null) {
 410                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 411                     () -> Collections.singletonList(
 412                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 413                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 414                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 415                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 416                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 417                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 418                                 }
 419                             }
 420                         })));
 421             }
 422             return;
 423         }
 424         // Get etherType value based on the IpPrefix address family type
 425         int etherType;
 426         try {
 427             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 428         } catch (IllegalArgumentException ex) {
 429             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 430             return;
 431         }
 432
 433         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 434         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 435             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 436                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 437                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 438                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 439                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 440                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 441                                     try {
 442                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 443                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 444                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 445                                                     TransactionAdapter.toWriteTransaction(tx),
 446                                                     txnObjects);
 447                                         } else {
 448                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 449                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 450                                         }
 451                                     } catch (NullPointerException e) {
 452                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 453                                                 vrfEntry.getDestPrefix(), e);
 454                                     }
 455                                 }
 456                             }
 457                         }
 458                     }
 459                 })), MAX_RETRIES);
 460         }
 461
 462         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 463         if (optVpnUuid.isPresent()) {
 464             String vpnUuid = optVpnUuid.get();
 465             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 466             if (interVpnLink != null) {
 467                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 468                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 469                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 470                         // This is an static route that points to the other endpoint of an InterVpnLink
 471                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 472                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 473                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 474                     }
 475                 });
 476             }
 477         }
 478     }
 479
 480     void refreshFibTables(String rd, String prefix) {
 481         InstanceIdentifier<VrfEntry> vrfEntryId =
 482                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 483                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 484         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 485         if (vrfEntry.isPresent()) {
 486             createFibEntries(vrfEntryId, vrfEntry.get());
 487         }
 488     }
 489
 490     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 491         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 492         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 493         prefixBuilder.setDpnId(lri.getDpnId());
 494         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 495         prefixBuilder.setIpAddress(lri.getPrefix());
 496         // Increment the refCount here
 497         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 498             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 499         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 500         if (!isPresentInList) {
 501             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 502             List<String> vpnInstanceNames = new ArrayList<>(nullToEmpty(lri.getVpnInstanceList()));
 503             vpnInstanceNames.add(vpnInstanceName);
 504             builder.setVpnInstanceList(vpnInstanceNames);
 505             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 506         } else {
 507             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 508         }
 509         return prefixBuilder.build();
 510     }
 511
 512     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 513             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 514         if (tx == null) {
 515             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 516                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 517                 "Error installing subnet route in FIB");
 518             return;
 519         }
 520         int etherType;
 521         try {
 522             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 523         } catch (IllegalArgumentException ex) {
 524             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 525             return;
 526         }
 527         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 528             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 529             synchronized (label.toString().intern()) {
 530                 LabelRouteInfo lri = getLabelRouteInfo(label);
 531                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 532
 533                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 534                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 535                                 fibUtil.getVpnInstanceOpData(rd);
 536                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 537                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 538                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 539                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 540                             }
 541                         }
 542                     }
 543                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 544                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 545                 }
 546             }
 547         });
 548         final List<InstructionInfo> instructions = new ArrayList<>();
 549         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 550             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 551         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 552         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 553         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 554                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 555         if (vrfEntry.getRoutePaths() != null) {
 556             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 557                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 558                     List<ActionInfo> actionsInfos = new ArrayList<>();
 559                     // reinitialize instructions list for LFIB Table
 560                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 561                     actionsInfos.add(new ActionPopMpls(etherType));
 562                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 563                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 564                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 565                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 566
 567                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 568                             NwConstants.ADD_FLOW, tx);
 569                 }
 570             }
 571         }
 572     }
 573
 574     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 575             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 576         List<MatchInfo> matches = new ArrayList<>();
 577
 578         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 579                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 580         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 581         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 582         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 583
 584         InetAddress destPrefix;
 585         try {
 586             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 587         } catch (UnknownHostException e) {
 588             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 589                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 590             return;
 591         }
 592
 593         // Match on VpnId and SubnetBroadCast IP address
 594         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 595         matches.add(MatchEthernetType.IPV4);
 596
 597         if (prefixLength != 0) {
 598             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 599         }
 600
 601         //Action is to drop the packet
 602         List<InstructionInfo> dropInstructions = new ArrayList<>();
 603         List<ActionInfo> actionsInfos = new ArrayList<>();
 604         actionsInfos.add(new ActionDrop());
 605         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 606
 607         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 608         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 609         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 610                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 611
 612         Flow flow = flowEntity.getFlowBuilder().build();
 613         String flowId = flowEntity.getFlowId();
 614         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 615         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 616
 617         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 618                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 619                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 620
 621         if (addOrRemove == NwConstants.ADD_FLOW) {
 622             tx.put(flowInstanceId,flow, true);
 623         } else {
 624             tx.delete(flowInstanceId);
 625         }
 626     }
 627
 628     /*
 629      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 630      * LportDispatcher table (via table 80)
 631      */
 632     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 633                                             final VrfEntry vrfEntry, int etherType) {
 634         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 635         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 636         // packet is commuted from Vpn2 to Vpn1.
 637         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 638         if (!interVpnLink.isActive()) {
 639             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 640                      interVpnLinkName, vrfEntry.getDestPrefix());
 641             return;
 642         }
 643
 644         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 645         if (!optLportTag.isPresent()) {
 646             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 647             return;
 648         }
 649
 650         Long lportTag = optLportTag.get();
 651         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 652         if (label == null) {
 653             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 654                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 655             return;
 656         }
 657         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 658         List<InstructionInfo> instructions = Arrays.asList(
 659             new InstructionApplyActions(actionsInfos),
 660             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 661                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 662                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 663                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 664             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 665         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 666         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 667
 668         for (BigInteger dpId : targetDpns) {
 669             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 670                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 671
 672             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 673                                /*writeTx*/null);
 674         }
 675     }
 676
 677
 678     /*
 679      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 680      */
 681     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 682                                                final VrfEntry vrfEntry, long vpnTag) {
 683         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 684         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 685             && vrfEntry.getRoutePaths().size() == 1);
 686         String destination = vrfEntry.getDestPrefix();
 687         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 688         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 689
 690         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 691         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 692         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 693         if (interVpnLink.getState().or(State.Error) != State.Active) {
 694             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 695                 destination, nextHop, interVpnLinkName);
 696             return;
 697         }
 698
 699         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 700         if (!optOtherEndpointLportTag.isPresent()) {
 701             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 702                 vpnUuid, interVpnLinkName);
 703             return;
 704         }
 705
 706         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 707         if (targetDpns.isEmpty()) {
 708             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 709                 vpnUuid, interVpnLinkName);
 710             return;
 711         }
 712
 713         String[] values = destination.split("/");
 714         String destPrefixIpAddress = values[0];
 715         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 716
 717         List<MatchInfo> matches = new ArrayList<>();
 718         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 719         matches.add(MatchEthernetType.IPV4);
 720
 721         if (prefixLength != 0) {
 722             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 723         }
 724
 725         List<Instruction> instructions =
 726             Arrays.asList(new InstructionWriteMetadata(
 727                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 728                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 729                             .L3VPN_SERVICE_INDEX)),
 730                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 731                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 732
 733         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 734         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 735         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 736             COOKIE_VM_FIB_TABLE, matches, instructions);
 737
 738         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 739             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 740
 741         for (BigInteger dpId : targetDpns) {
 742
 743             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 744                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 745                 dpId, interVpnLink.getInterVpnLinkName());
 746
 747             mdsalManager.installFlow(dpId, flowEntity);
 748         }
 749     }
 750
 751     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 752         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 753         String localNextHopIP = vrfEntry.getDestPrefix();
 754         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 755         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 756         if (localNextHopInfo == null) {
 757             boolean localNextHopSeen = false;
 758             List<Routes> vpnExtraRoutes = null;
 759             String rdPrefixKey = localNextHopIP + FibConstants.SEPARATOR + rd;
 760             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 761             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 762             synchronized (rdPrefixKey.intern()) {
 763                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 764                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 765                         vpnName, usedRds, localNextHopIP);
 766                 if (LOG.isDebugEnabled()) {
 767                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 768                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 769                 }
 770
 771                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 772                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 773                     String ipPrefix;
 774                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 775                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 776                     } else {
 777                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 778                     }
 779                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 780                             ipPrefix);
 781                     if (localNextHopInfoLocal != null) {
 782                         localNextHopSeen = true;
 783                         BigInteger dpnId =
 784                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 785                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 786                         returnLocalDpnId.add(dpnId);
 787                     }
 788                 }
 789             }
 790             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 791                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 792                 if (optionalLabel.isPresent()) {
 793                     Long label = optionalLabel.get();
 794                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 795                     synchronized (label.toString().intern()) {
 796                         LabelRouteInfo lri = getLabelRouteInfo(label);
 797                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 798                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 799                                     fibUtil.getVpnInstanceOpData(rd);
 800                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 801                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 802                                 if (nullToEmpty(lri.getVpnInstanceList()).contains(vpnInstanceName)) {
 803                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 804                                     localNextHopIP = lri.getPrefix();
 805                                 } else {
 806                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 807                                     localNextHopIP = lri.getPrefix();
 808                                 }
 809                             }
 810                             if (localNextHopInfo != null) {
 811                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 812                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 813                                 if (vpnExtraRoutes.isEmpty()) {
 814                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 815                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 816                                     returnLocalDpnId.add(dpnId);
 817                                 } else {
 818                                     for (Routes extraRoutes : vpnExtraRoutes) {
 819                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 820                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 821                                         returnLocalDpnId.add(dpnId);
 822                                     }
 823                                 }
 824                             }
 825                         }
 826                     }
 827                 }
 828             }
 829             if (returnLocalDpnId.isEmpty()) {
 830                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 831             }
 832         } else {
 833             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 834                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 835             if (dpnId != null) {
 836                 returnLocalDpnId.add(dpnId);
 837             }
 838         }
 839         return returnLocalDpnId;
 840     }
 841
 842     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 843                                                 final Long vpnId, final String rd,
 844                                                 final VrfEntry vrfEntry,
 845                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 846                                                 int etherType) {
 847         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 848         if (localNextHopInfo != null) {
 849             long groupId;
 850             long localGroupId;
 851             final BigInteger dpnId = localNextHopInfo.getDpnId();
 852             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 853                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 854                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 855                 return dpnId;
 856             }
 857             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 858                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 859                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 860                 return dpnId;
 861             }
 862             if (!isVpnPresentInDpn(rd, dpnId)) {
 863                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 864                         vpnId, rd, dpnId.toString());
 865                 return BigInteger.ZERO;
 866             }
 867             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 868             String prefix = vrfEntry.getDestPrefix();
 869             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 870             //The loadbalancing group is created only if the extra route has multiple nexthops
 871             //to avoid loadbalancing the discovered routes
 872             if (vpnExtraRoutes != null && routes != null) {
 873                 if (vpnExtraRoutes.size() > 1) {
 874                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 875                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 876                 } else {
 877                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 878                     localGroupId = groupId;
 879                 }
 880             } else {
 881                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 882                         gwMacAddress);
 883                 localGroupId = groupId;
 884             }
 885             if (groupId == FibConstants.INVALID_GROUP_ID) {
 886                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 887                         prefix, rd, interfaceName, dpnId.toString());
 888                 return BigInteger.ZERO;
 889             }
 890             final List<InstructionInfo> instructions = Collections.singletonList(
 891                     new InstructionApplyActions(
 892                             Collections.singletonList(new ActionGroup(groupId))));
 893             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 894                     new InstructionApplyActions(
 895                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 896             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 897             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 898             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 899             jobCoordinator.enqueueJob(jobKey,
 900                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 901                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 902                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 903                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 904                         optLabel.ifPresent(label -> {
 905                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 906                                 LOG.debug(
 907                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 908                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 909                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 910                                         nextHopAddressList);
 911                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 912                                         NwConstants.ADD_FLOW, tx);
 913                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 914                             } else {
 915                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 916                                                 + "route. LFib and Terminating table entries will not be created.",
 917                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 918                             }
 919                         });
 920                     }
 921                 })));
 922             return dpnId;
 923         }
 924         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 925                 vpnName);
 926         return BigInteger.ZERO;
 927     }
 928
 929     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 930         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 931         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 932         return dpnInVpn.isPresent();
 933     }
 934
 935     @Nullable
 936     private LabelRouteInfo getLabelRouteInfo(Long label) {
 937         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 938             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 939         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 940         if (opResult.isPresent()) {
 941             return opResult.get();
 942         }
 943         return null;
 944     }
 945
 946     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 947             @Nullable TypedWriteTransaction<Operational> tx) {
 948         if (lri == null) {
 949             return true;
 950         }
 951
 952         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 953         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 954             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 955
 956         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 957             ? lri.getVpnInstanceList() : new ArrayList<>();
 958         if (vpnInstancesList.contains(vpnInstanceName)) {
 959             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 960             vpnInstancesList.remove(vpnInstanceName);
 961         }
 962         if (vpnInstancesList.isEmpty()) {
 963             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 964             if (tx != null) {
 965                 tx.delete(lriId);
 966             } else {
 967                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 968             }
 969             return true;
 970         } else {
 971             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 972             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 973             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 974         }
 975         return false;
 976     }
 977
 978     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 979                                       TypedWriteTransaction<Configuration> tx) {
 980         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 981
 982         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 983
 984         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 985             dpId, label, groupId);
 986     }
 987
 988     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 989                                                 TypedWriteTransaction<Configuration> tx) {
 990         List<MatchInfo> mkMatches = new ArrayList<>();
 991
 992         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 993             destDpId, label, actionsInfos);
 994
 995         // Matching metadata
 996         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 997         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 998
 999         List<InstructionInfo> mkInstructions = new ArrayList<>();
1000         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1001
1002         FlowEntity terminatingServiceTableFlowEntity =
1003             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1004             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1005                 String.format("%s:%d", "TST Flow Entry ", label),
1006             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1007
1008         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1009
1010         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1011
1012         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1013         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1014             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1015             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1016             .child(Flow.class, flowKey).build();
1017         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1018     }
1019
1020     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1021         FlowEntity flowEntity;
1022         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1023         List<MatchInfo> mkMatches = new ArrayList<>();
1024         // Matching metadata
1025         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1026         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1027             NwConstants.INTERNAL_TUNNEL_TABLE,
1028             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1029             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1030             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1031         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1032         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1033         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1034             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1035             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1036
1037         tx.delete(flowInstanceId);
1038         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1039     }
1040
1041     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1042         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1043         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1044         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1045         boolean shouldUpdateNonEcmpLocalNextHop = true;
1046         if (localNextHopInfo == null) {
1047             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1048             if (usedRds.size() > 1) {
1049                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1050                         vrfEntry.getDestPrefix(), vpnName, rd);
1051                 return returnLocalDpnId;
1052             }
1053             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1054             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1055             //in the vpn
1056             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1057                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1058             if (extraRouteOptional.isPresent()) {
1059                 Routes extraRoute = extraRouteOptional.get();
1060                 String ipPrefix;
1061                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1062                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1063                 } else {
1064                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1065                 }
1066                 if (extraRoute.getNexthopIpList().size() > 1) {
1067                     shouldUpdateNonEcmpLocalNextHop = false;
1068                 }
1069                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1070                 if (localNextHopInfo != null) {
1071                     String localNextHopIP = localNextHopInfo.getIpAddress();
1072                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1073                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1074                     if (!dpnId.equals(BigInteger.ZERO)) {
1075                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1076                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1077                         returnLocalDpnId.add(dpnId);
1078                     }
1079                 } else {
1080                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1081                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1082                 }
1083             }
1084
1085             if (localNextHopInfo == null) {
1086                 /* Imported VRF entry */
1087                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1088                 if (optionalLabel.isPresent()) {
1089                     Long label = optionalLabel.get();
1090                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1091                     LabelRouteInfo lri = getLabelRouteInfo(label);
1092                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1093                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1094                         prefixBuilder.setDpnId(lri.getDpnId());
1095                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1096                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1097                         if (!dpnId.equals(BigInteger.ZERO)) {
1098                             returnLocalDpnId.add(dpnId);
1099                         }
1100                     }
1101                 }
1102             }
1103
1104         } else {
1105             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1106             String localNextHopIP = localNextHopInfo.getIpAddress();
1107             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1108                     shouldUpdateNonEcmpLocalNextHop);
1109             if (!dpnId.equals(BigInteger.ZERO)) {
1110                 returnLocalDpnId.add(dpnId);
1111             }
1112         }
1113
1114         return returnLocalDpnId;
1115     }
1116
1117     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1118             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1119             boolean shouldUpdateNonEcmpLocalNextHop) {
1120         if (localNextHopInfo != null) {
1121             final BigInteger dpnId = localNextHopInfo.getDpnId();
1122             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1123                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1124                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1125                 return dpnId;
1126             }
1127             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1128                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1129                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1130                 return dpnId;
1131             }
1132
1133             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1134                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1135                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1136                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1137                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1138                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1139                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1140                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1141                                         NwConstants.DEL_FLOW, tx);
1142                                 removeTunnelTableEntry(dpnId, label, tx);
1143                             });
1144                         }
1145                     }
1146                 })));
1147             //TODO: verify below adjacency call need to be optimized (?)
1148             //In case of the removal of the extra route, the loadbalancing group is updated
1149             if (shouldUpdateNonEcmpLocalNextHop) {
1150                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1151             }
1152             return dpnId;
1153         }
1154         return BigInteger.ZERO;
1155     }
1156
1157     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1158             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1159         if (tx == null) {
1160             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1161                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1162                 "Error creating remote FIB entry");
1163             return;
1164         }
1165
1166         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1167         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1168                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1169
1170         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1171         if (adjacencyResults.isEmpty()) {
1172             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1173                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1174             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1175             return;
1176         }
1177
1178         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1179         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1180                 vpnName, usedRds, vrfEntry.getDestPrefix());
1181         // create loadbalancing groups for extra routes only when the extra route is present behind
1182         // multiple VMs
1183         if (!vpnExtraRoutes.isEmpty()) {
1184             List<InstructionInfo> instructions = new ArrayList<>();
1185             // Obtain the local routes for this particular dpn.
1186             java.util.Optional<Routes> routes = vpnExtraRoutes
1187                     .stream()
1188                     .filter(route -> {
1189                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1190                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1191                         if (prefixToInterface == null) {
1192                             return false;
1193                         }
1194                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1195                     }).findFirst();
1196             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1197                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1198             if (groupId == FibConstants.INVALID_GROUP_ID) {
1199                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1200                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1201                 return;
1202             }
1203             List<ActionInfo> actionInfos =
1204                     Collections.singletonList(new ActionGroup(groupId));
1205             instructions.add(new InstructionApplyActions(actionInfos));
1206             String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1207             jobCoordinator.enqueueJob(jobKey,
1208                 () ->  Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1209                     baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1210                             NwConstants.ADD_FLOW, txn, null);
1211                 })));
1212         } else {
1213             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry,
1214                 TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null);
1215         }
1216
1217         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1218     }
1219
1220     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1221     /* Get interface info from prefix to interface mapping;
1222         Use the interface info to get the corresponding vpn interface op DS entry,
1223         remove the adjacency corresponding to this fib entry.
1224         If adjacency removed is the last adjacency, clean up the following:
1225          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1226          - prefix to interface entry
1227          - vpn interface op DS
1228      */
1229         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1230         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1231         if (prefixInfo == null) {
1232             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1233             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1234             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1235             if (extraRoute != null) {
1236                 for (String nextHopIp : nullToEmpty(extraRoute.getNexthopIpList())) {
1237                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1238                     if (nextHopIp != null) {
1239                         String ipPrefix;
1240                         if (isIpv4Address(nextHopIp)) {
1241                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1242                         } else {
1243                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1244                         }
1245                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1246                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1247                     }
1248                 }
1249             }
1250             if (prefixInfo == null) {
1251                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1252                 if (optionalLabel.isPresent()) {
1253                     Long label = optionalLabel.get();
1254                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1255                     LabelRouteInfo lri = getLabelRouteInfo(label);
1256                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1257                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1258                         prefixBuilder.setDpnId(lri.getDpnId());
1259                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1260                         prefixBuilder.setIpAddress(lri.getPrefix());
1261                         prefixInfo = prefixBuilder.build();
1262                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1263                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1264                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1265                     }
1266                 }
1267             }
1268         } else {
1269             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1270         }
1271     }
1272
1273     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1274                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1275
1276         if (prefixInfo == null) {
1277             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1278                     vrfEntry.getDestPrefix(), vpnId, rd);
1279             return; //Don't have any info for this prefix (shouldn't happen); need to return
1280         }
1281
1282         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1283             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1284                     vrfEntry.getDestPrefix(), vpnId, rd);
1285             return;
1286         }
1287
1288         String ifName = prefixInfo.getVpnInterfaceName();
1289         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1290             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1291     }
1292
1293     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1294         Prefixes prefixInfo;
1295         Long vpnId;
1296         String rd;
1297         VrfEntry vrfEntry;
1298         Routes extraRoute;
1299
1300         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1301                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1302             this.prefixInfo = prefixInfo;
1303             this.vpnId = vpnId;
1304             this.rd = rd;
1305             this.vrfEntry = vrfEntry;
1306             this.extraRoute = extraRoute;
1307         }
1308
1309         @Override
1310         public List<ListenableFuture<Void>> call() {
1311             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1312             // to call the respective helpers.
1313             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1314                 //First Cleanup LabelRouteInfo
1315                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1316                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1317                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1318                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1319                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1320                         synchronized (label.toString().intern()) {
1321                             LabelRouteInfo lri = getLabelRouteInfo(label);
1322                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1323                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1324                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1325                                         fibUtil.getVpnInstanceOpData(rd);
1326                                 String vpnInstanceName = "";
1327                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1328                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1329                                 }
1330                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1331                                 if (lriRemoved) {
1332                                     String parentRd = lri.getParentVpnRd();
1333                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1334                                             parentRd, vrfEntry.getDestPrefix()));
1335                                 }
1336                             } else {
1337                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1338                                         rd, vrfEntry.getDestPrefix()));
1339                             }
1340                         }
1341                     });
1342                 }
1343                 String ifName = prefixInfo.getVpnInterfaceName();
1344                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1345                 String vpnName = null;
1346
1347                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1348                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1349                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1350                     if (vpnNameOpt.isPresent()) {
1351                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1352                     }
1353                 }
1354                 if (optVpnName.isPresent()) {
1355                     vpnName = optVpnName.get();
1356                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1357                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1358                     if (opVpnInterface.isPresent()) {
1359                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1360                         if (vpnId != associatedVpnId) {
1361                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1362                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1363                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1364                             return;
1365                         } else {
1366                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1367                                     vrfEntry.getDestPrefix(), associatedVpnId);
1368                         }
1369                     }
1370                 }
1371                 if (extraRoute != null) {
1372                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1373                     //Only one used Rd present in case of removal event
1374                     String usedRd = usedRds.get(0);
1375                     if (optVpnName.isPresent()) {
1376                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1377                                         vrfEntry.getDestPrefix()));
1378                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1379                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1380                     }
1381                 }
1382                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1383             }));
1384         }
1385     }
1386
1387     /**
1388      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1389      * Remove Adjacency from VPNInterfaceOpData.
1390      * if Adjacency != primary.
1391      * if Adjacency == primary , then mark it for deletion.
1392      * Remove entire VPNinterfaceOpData Entry.
1393      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1394      * @param vrfEntry - VrfEntry removed
1395      * @param ifName - Interface name from VRFentry
1396      * @param vpnName - VPN name of corresponding VRF
1397      * @param tx - ReadWrite Tx
1398      */
1399     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1400                                                           TypedReadWriteTransaction<Operational> tx)
1401             throws ExecutionException, InterruptedException {
1402         InstanceIdentifier<Adjacency> adjacencyIid =
1403                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1404         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1405         if (adjacencyOptional.isPresent()) {
1406             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1407                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1408             } else {
1409                 tx.merge(adjacencyIid,
1410                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1411             }
1412         }
1413
1414         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1415
1416         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1417             return;
1418         }
1419
1420         if (nullToEmpty(optAdjacencies.get().getAdjacency()).size() <= 2
1421                 && nullToEmpty(optAdjacencies.get().getAdjacency()).stream().allMatch(adjacency ->
1422                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1423                         && adjacency.isMarkedForDeletion() != null
1424                         && adjacency.isMarkedForDeletion()
1425         )) {
1426             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1427             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1428         }
1429     }
1430
1431     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1432         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1433         final String rd = vrfTableKey.getRouteDistinguisher();
1434         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1435         if (vpnInstance == null) {
1436             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1437             return;
1438         }
1439         final Collection<VpnToDpnList> vpnToDpnList;
1440         if (vrfEntry.getParentVpnRd() != null
1441                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1442             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1443             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1444             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1445                     vpnInstance.getVpnToDpnList();
1446             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1447                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1448         } else {
1449             vpnToDpnList = vpnInstance.getVpnToDpnList();
1450         }
1451
1452         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1453         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1454         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1455         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1456         if (subnetRoute != null) {
1457             long elanTag = subnetRoute.getElantag();
1458             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1459                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1460             if (vpnToDpnList != null) {
1461                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1462                     () -> Collections.singletonList(
1463                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1464                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1465
1466                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1467                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1468                                     TransactionAdapter.toWriteTransaction(tx), null);
1469                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1470                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1471                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1472                                 }
1473
1474                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1475                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1476                             }
1477                         })));
1478             }
1479             optionalLabel.ifPresent(label -> {
1480                 synchronized (label.toString().intern()) {
1481                     LabelRouteInfo lri = getLabelRouteInfo(label);
1482                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1483                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1484                                 fibUtil.getVpnInstanceOpData(rd);
1485                         String vpnInstanceName = "";
1486                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1487                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1488                         }
1489                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1490                         if (lriRemoved) {
1491                             String parentRd = lri.getParentVpnRd();
1492                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1493                                     parentRd, vrfEntry.getDestPrefix()));
1494                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1495                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1496                         }
1497                     } else {
1498                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1499                                 rd, vrfEntry.getDestPrefix()));
1500                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1501                                 label, rd, vrfEntry.getDestPrefix());
1502                     }
1503                 }
1504             });
1505             return;
1506         }
1507
1508         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1509             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1510         if (vpnToDpnList != null) {
1511             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1512                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1513             String jobKey;
1514             Optional<Routes> extraRouteOptional;
1515             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1516             if (usedRds != null && !usedRds.isEmpty()) {
1517                 if (usedRds.size() > 1) {
1518                     LOG.error("The extra route prefix is still present in some DPNs");
1519                     return ;
1520                 } else {
1521                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1522                     //is not present in any other DPN
1523                     extraRouteOptional = VpnExtraRouteHelper
1524                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1525                 }
1526             } else {
1527                 extraRouteOptional = Optional.absent();
1528             }
1529
1530             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1531                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1532                     if (localDpnIdList.size() <= 0) {
1533                         for (VpnToDpnList curDpn : vpnToDpnList) {
1534                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1535                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1536                                 TransactionAdapter.toWriteTransaction(tx));
1537                         }
1538                     } else {
1539                         for (BigInteger localDpnId : localDpnIdList) {
1540                             for (VpnToDpnList curDpn : vpnToDpnList) {
1541                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1542                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1543                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1544                                         TransactionAdapter.toWriteTransaction(tx));
1545                                 }
1546                             }
1547                         }
1548                     }
1549                     nextHopManager.removeNextHopPointer(nextHopManager
1550                             .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1551                     nextHopManager.removeNextHopPointer(nextHopManager
1552                             .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1553                 })), MAX_RETRIES);
1554         }
1555
1556         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1557         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1558         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1559
1560         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1561         // of the interVpnLink.
1562         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1563         if (optVpnUuid.isPresent()) {
1564             String vpnUuid = optVpnUuid.get();
1565             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1566                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1567                 if (optInterVpnLink.isPresent()) {
1568                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1569                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1570                         // This is route that points to the other endpoint of an InterVpnLink
1571                         // In that case, we should look for the FIB table pointing to
1572                         // LPortDispatcher table and remove it.
1573                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1574                     }
1575                 }
1576             });
1577         }
1578
1579     }
1580
1581     private void makeLFibTableEntry(BigInteger dpId, long label, @Nullable List<InstructionInfo> instructions,
1582                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1583         if (tx == null) {
1584             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1585                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1586                 "Error making LFIB table entry");
1587             return;
1588         }
1589
1590         List<MatchInfo> matches = new ArrayList<>();
1591         matches.add(MatchEthernetType.MPLS_UNICAST);
1592         matches.add(new MatchMplsLabel(label));
1593
1594         // Install the flow entry in L3_LFIB_TABLE
1595         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1596
1597         FlowEntity flowEntity;
1598         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1599             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1600         Flow flow = flowEntity.getFlowBuilder().build();
1601         String flowId = flowEntity.getFlowId();
1602         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1603         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1604         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1605             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1606             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1607
1608         if (addOrRemove == NwConstants.ADD_FLOW) {
1609             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1610         } else {
1611             tx.delete(flowInstanceId);
1612         }
1613
1614         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1615             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1616     }
1617
1618     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1619                                     final FutureCallback<List<Void>> callback) {
1620         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1621         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1622             () -> {
1623                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1624                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1625                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1626                         LogicalDatastoreType.CONFIGURATION, id);
1627                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1628                 if (!vrfTable.isPresent()) {
1629                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1630                     if (callback != null) {
1631                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1632                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1633                     }
1634                     return futures;
1635                 }
1636                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1637                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1638                         for (final VrfEntry vrfEntry : nullToEmpty(vrfTable.get().getVrfEntry())) {
1639                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1640                             if (subnetRoute != null) {
1641                                 long elanTag = subnetRoute.getElantag();
1642                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1643                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1644                                         tx);
1645                                 continue;
1646                             }
1647                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1648                             if (routerInt != null) {
1649                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1650                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1651                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1652                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1653                                         NwConstants.ADD_FLOW);
1654                                 continue;
1655                             }
1656                             //Handle local flow creation for imports
1657                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1658                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1659                                 if (optionalLabel.isPresent()) {
1660                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1661                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1662                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1663                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1664                                             try {
1665                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1666                                                         vrfEntry.getDestPrefix());
1667                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1668                                             } catch (IllegalArgumentException ex) {
1669                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1670                                                         vrfEntry.getDestPrefix());
1671                                             }
1672                                             continue;
1673                                         }
1674                                     }
1675                                 }
1676                             }
1677                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1678                                     vrfEntry, dpnId);
1679                             if (shouldCreateRemoteFibEntry) {
1680                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1681                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1682                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1683                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1684                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1685                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1686                                 } else {
1687                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1688                                             vrfEntry, tx);
1689                                 }
1690                             }
1691                         }
1692                     }));
1693                     if (callback != null) {
1694                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1695                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1696                     }
1697                 }
1698                 return futures;
1699             });
1700     }
1701
1702     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1703                                             final String localNextHopIp, final String remoteNextHopIp) {
1704         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1705             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1706         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1707         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1708         List<SubTransaction> txnObjects =  new ArrayList<>();
1709         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1710         if (vrfTable.isPresent()) {
1711             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1712                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1713                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1714                         nullToEmpty(vrfTable.get().getVrfEntry()).stream()
1715                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1716                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1717                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1718                     }
1719                 })));
1720         }
1721     }
1722
1723     public void manageRemoteRouteOnDPN(final boolean action,
1724                                        final BigInteger localDpnId,
1725                                        final long vpnId,
1726                                        final String rd,
1727                                        final String destPrefix,
1728                                        final String destTepIp,
1729                                        final long label) {
1730         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1731
1732         if (vpnInstance == null) {
1733             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1734             return;
1735         }
1736
1737         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1738             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1739                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1740                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1741                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1742                     if (vrfEntry == null) {
1743                         return;
1744                     }
1745                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1746                             action, localDpnId, vpnId, rd, destPrefix);
1747                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1748                     VrfEntry modVrfEntry;
1749                     if (routePathList == null || routePathList.isEmpty()) {
1750                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1751                                 Collections.singletonList(destTepIp),
1752                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1753                     } else {
1754                         modVrfEntry = vrfEntry;
1755                     }
1756
1757                     if (action) {
1758                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1759                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1760                                 modVrfEntry, tx);
1761                     } else {
1762                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1763                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1764                                 vrfEntry.getDestPrefix());
1765                         if (usedRds.size() > 1) {
1766                             LOG.debug("The extra route prefix is still present in some DPNs");
1767                             return;
1768                         }
1769                         //Is this fib route an extra route? If yes, get the nexthop which would be
1770                         //an adjacency in the vpn
1771                         Optional<Routes> extraRouteOptional = Optional.absent();
1772                         if (usedRds.size() != 0) {
1773                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1774                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1775                                     usedRds.get(0), vrfEntry.getDestPrefix());
1776                         }
1777                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1778                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1779                     }
1780                 }
1781             })));
1782     }
1783
1784     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1785                                  final FutureCallback<List<Void>> callback) {
1786         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1787         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1788             () -> {
1789                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1790                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1791                 List<SubTransaction> txnObjects =  new ArrayList<>();
1792                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1793                         LogicalDatastoreType.CONFIGURATION, id);
1794                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1795                 if (vrfTable.isPresent()) {
1796                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1797                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1798                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1799                             for (final VrfEntry vrfEntry : nullToEmpty(vrfTable.get().getVrfEntry())) {
1800                                 /* parentRd is only filled for external PNF cases where the interface on the external
1801                                  * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1802                                  * #fibUtil.isInterfacePresentInDpn().
1803                                 * */
1804                                 String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1805                                         : rd;
1806                                 /* Handle subnet routes here */
1807                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1808                                 if (subnetRoute != null && !fibUtil
1809                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1810                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1811                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1812                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1813                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1814                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1815                                     if (routePaths != null) {
1816                                         for (RoutePaths routePath : routePaths) {
1817                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1818                                                     DEFAULT_FIB_FLOW_PRIORITY,
1819                                                     NwConstants.DEL_FLOW, tx);
1820                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1821                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1822                                                     vrfEntry.getDestPrefix());
1823                                         }
1824                                     }
1825                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1826                                             NwConstants.DEL_FLOW, tx);
1827                                     continue;
1828                                 }
1829                                 // ping responder for router interfaces
1830                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1831                                 if (routerInt != null) {
1832                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1833                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1834                                             routerInt.getMacAddress());
1835                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1836                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1837                                             NwConstants.DEL_FLOW);
1838                                     continue;
1839                                 }
1840
1841                                 //Handle local flow deletion for imports
1842                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1843                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1844                                     if (optionalLabel.isPresent()) {
1845                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1846                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1847                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1848                                                 lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1849                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1850                                         }
1851                                     }
1852                                 }
1853
1854                                 // Passing null as we don't know the dpn
1855                                 // to which prefix is attached at this point
1856                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1857                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1858                                 Optional<Routes> extraRouteOptional;
1859                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1860                                 //an adjacency in the vpn
1861                                 if (usedRds != null && !usedRds.isEmpty()) {
1862                                     if (usedRds.size() > 1) {
1863                                         LOG.error("The extra route prefix is still present in some DPNs");
1864                                         return;
1865                                     } else {
1866                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1867                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1868
1869                                     }
1870                                 } else {
1871                                     extraRouteOptional = Optional.absent();
1872                                 }
1873                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1874                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1875                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1876                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1877                                 } else {
1878                                     if (subnetRoute == null || !fibUtil
1879                                             .isInterfacePresentInDpn(parentRd, dpnId)) {
1880                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1881                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1882                                             TransactionAdapter.toWriteTransaction(tx));
1883                                     }
1884                                 }
1885                             }
1886                         }));
1887                     }
1888                     if (callback != null) {
1889                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1890                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1891                     }
1892                 } else {
1893                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1894                 }
1895                 return futures;
1896             });
1897
1898     }
1899
1900     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1901                                            final String localNextHopIp, final String remoteNextHopIp) {
1902         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1903                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1904             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1905         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1906         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1907         List<SubTransaction> txnObjects =  new ArrayList<>();
1908         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1909         if (vrfTable.isPresent()) {
1910             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1911                 () -> {
1912                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1913                         return Collections.singletonList(
1914                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1915                                 tx -> nullToEmpty(vrfTable.get().getVrfEntry()).stream()
1916                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1917                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1918                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1919                                         txnObjects))));
1920                     }
1921                 });
1922         }
1923     }
1924
1925     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1926         InstanceIdentifierBuilder<VrfTables> idBuilder =
1927             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1928         return idBuilder.build();
1929     }
1930
1931     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1932         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1933                 .FLOWID_SEPARATOR + nextHop;
1934     }
1935
1936     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1937         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1938                 + tableMiss + FLOWID_PREFIX;
1939     }
1940
1941     @Nullable
1942     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1943         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1944             .child(VrfTables.class, new VrfTablesKey(rd))
1945             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1946         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1947         if (vrfEntry.isPresent()) {
1948             return vrfEntry.get();
1949         }
1950         return null;
1951     }
1952
1953     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1954                                              final String vpnName,
1955                                              final VrfEntry vrfEntry) {
1956         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1957
1958         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1959         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1960
1961         if (targetDpns.isEmpty()) {
1962             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1963             return;
1964         }
1965
1966         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1967         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1968
1969         // delete from FIB
1970         //
1971         optNextHop.ifPresent(nextHop -> {
1972             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1973             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1974             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1975                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1976
1977             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1978             for (BigInteger dpId : targetDpns) {
1979                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1980                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1981
1982                 mdsalManager.removeFlow(dpId, flow);
1983             }
1984         });
1985
1986         // delete from LFIB
1987         //
1988         optLabel.ifPresent(label -> {
1989             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1990
1991             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1992                 for (BigInteger dpId : targetDpns) {
1993                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1994                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1995                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1996                             tx);
1997                 }
1998             }), LOG, "Error removing flows");
1999         });
2000     }
2001
2002     private boolean isPrefixAndNextHopPresentInLri(String prefix,
2003             List<String> nextHopAddressList, LabelRouteInfo lri) {
2004         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2005                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2006     }
2007
2008     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2009         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2010             return true;
2011         }
2012
2013         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2014         if (prefix != null) {
2015             BigInteger prefixDpnId = prefix.getDpnId();
2016             if (dpnId.equals(prefixDpnId)) {
2017                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2018                         vrfEntry, dpnId);
2019                 return false;
2020             }
2021         }
2022         return true;
2023     }
2024 }