fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.math.BigInteger;
  23 import java.net.InetAddress;
  24 import java.net.UnknownHostException;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Collection;
  28 import java.util.Collections;
  29 import java.util.List;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.CopyOnWriteArrayList;
  32 import java.util.concurrent.ExecutionException;
  33 import javax.annotation.PostConstruct;
  34 import javax.inject.Inject;
  35 import javax.inject.Singleton;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  40 import org.opendaylight.genius.infra.Datastore.Configuration;
  41 import org.opendaylight.genius.infra.Datastore.Operational;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  44 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.TransactionAdapter;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.mdsalutil.ActionInfo;
  49 import org.opendaylight.genius.mdsalutil.FlowEntity;
  50 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  51 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  52 import org.opendaylight.genius.mdsalutil.MatchInfo;
  53 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  54 import org.opendaylight.genius.mdsalutil.NWUtil;
  55 import org.opendaylight.genius.mdsalutil.NwConstants;
  56 import org.opendaylight.genius.mdsalutil.UpgradeState;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  63 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  69 import org.opendaylight.genius.utils.ServiceIndex;
  70 import org.opendaylight.genius.utils.batching.SubTransaction;
  71 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  72 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  73 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  74 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  75 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  76 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  77 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  78 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  79 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  80 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 114 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 115 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119
 120 @Singleton
 121 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 122
 123     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 124     private static final String FLOWID_PREFIX = "L3.";
 125     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 126     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 127     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 128     private static final int LFIB_INTERVPN_PRIORITY = 15;
 129     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 130     private static final int MAX_RETRIES = 3;
 131     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 136     private final IMdsalApiManager mdsalManager;
 137     private final NexthopManager nextHopManager;
 138     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 139     private final BaseVrfEntryHandler baseVrfEntryHandler;
 140     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final IElanService elanManager;
 143     private final FibUtil fibUtil;
 144     private final InterVpnLinkCache interVpnLinkCache;
 145     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 146     private final UpgradeState upgradeState;
 147     private final DataTreeEventCallbackRegistrar eventCallbacks;
 148
 149     @Inject
 150     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 151                             final NexthopManager nexthopManager,
 152                             final IElanService elanManager,
 153                             final BaseVrfEntryHandler vrfEntryHandler,
 154                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 155                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 156                             final JobCoordinator jobCoordinator,
 157                             final FibUtil fibUtil,
 158                             final InterVpnLinkCache interVpnLinkCache,
 159                             final UpgradeState upgradeState,
 160                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 161         super(VrfEntry.class, VrfEntryListener.class);
 162         this.dataBroker = dataBroker;
 163         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 164         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 165         this.mdsalManager = mdsalApiManager;
 166         this.nextHopManager = nexthopManager;
 167         this.elanManager = elanManager;
 168         this.baseVrfEntryHandler = vrfEntryHandler;
 169         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 170         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 171         this.jobCoordinator = jobCoordinator;
 172         this.fibUtil = fibUtil;
 173         this.interVpnLinkCache = interVpnLinkCache;
 174         this.upgradeState = upgradeState;
 175         this.eventCallbacks = eventCallbacks;
 176     }
 177
 178     @Override
 179     @PostConstruct
 180     public void init() {
 181         LOG.info("{} init", getClass().getSimpleName());
 182         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 183     }
 184
 185     @Override
 186     @SuppressWarnings("checkstyle:IllegalCatch")
 187     public void close() {
 188         closeables.forEach(c -> {
 189             try {
 190                 c.close();
 191             } catch (Exception e) {
 192                 LOG.warn("Error closing {}", c, e);
 193             }
 194         });
 195     }
 196
 197     @Override
 198     protected VrfEntryListener getDataTreeChangeListener() {
 199         return VrfEntryListener.this;
 200     }
 201
 202     @Override
 203     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 204         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 205     }
 206
 207     @Override
 208     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 209         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 210         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 211         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 212                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 213         addFibEntries(identifier, vrfEntry, rd);
 214         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 215                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 216     }
 217
 218     //This method is temporary. Eventually Factory design pattern will be used to get
 219     // right VrfEntryhandle and invoke its methods.
 220     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 221         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 222             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 223             return;
 224         }
 225         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 226             LOG.info("EVPN flows need to be programmed.");
 227             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 228                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 229             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 230             closeables.add(evpnVrfEntryHandler);
 231             return;
 232         }
 233         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 234         if (routerInt != null) {
 235             // ping responder for router interfaces
 236             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 237             return;
 238         }
 239         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 240             createFibEntries(identifier, vrfEntry);
 241             return;
 242         }
 243     }
 244
 245     @Override
 246     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 247         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 248         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 249         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 250                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 251         removeFibEntries(identifier, vrfEntry, rd);
 252         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 253             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 254     }
 255
 256     //This method is temporary. Eventually Factory design pattern will be used to get
 257     // right VrfEntryhandle and invoke its methods.
 258     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 259         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 260             LOG.info("EVPN flows to be deleted");
 261             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 262                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 263             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 264             closeables.add(evpnVrfEntryHandler);
 265             return;
 266         }
 267         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 268         if (routerInt != null) {
 269             // ping responder for router interfaces
 270             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 271             return;
 272         }
 273         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 274             deleteFibEntries(identifier, vrfEntry);
 275             return;
 276         }
 277         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 278             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 279             return;
 280         }
 281     }
 282
 283     @Override
 284     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 285     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 286     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 287     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 288         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 289         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 290         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 291                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 292         // Handle BGP Routes first
 293         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 294             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 295             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 296                     rd, update.getDestPrefix(), update.getRoutePaths());
 297             return;
 298         }
 299
 300         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 301             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 302             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 303             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 304
 305             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 306             //updated vrf entry route path is null. This can happen during tunnel events.
 307             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 308             List<String> usedRds = new ArrayList<>();
 309             if (optVpnInstance.isPresent()) {
 310                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 311                         update.getDestPrefix());
 312             }
 313             // If original VRF Entry had nexthop null , but update VRF Entry
 314             // has nexthop , route needs to be created on remote Dpns
 315             if (originalRoutePath == null || originalRoutePath.isEmpty()
 316                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 317                 // TODO(vivek): Though ugly, Not handling this code now, as each
 318                 // tep add event will invoke flow addition
 319                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 320                         + " This event is IGNORED here.", update.getDestPrefix());
 321                 return;
 322             }
 323
 324             // If original VRF Entry had valid nexthop , but update VRF Entry
 325             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 326             if (updateRoutePath == null || updateRoutePath.isEmpty()
 327                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 328                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 329                         + "This event is IGNORED here.", update.getDestPrefix());
 330                 return;
 331             }
 332             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 333             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 334             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 335             List<ListenableFuture<Void>> futures = new ArrayList<>();
 336             ListenableFuture<Void> configFuture =
 337                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 338                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 339                         nextHopsRemoved.parallelStream()
 340                             .forEach(nextHopRemoved -> {
 341                                 try {
 342                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 343                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 344                                 } catch (ExecutionException | InterruptedException e) {
 345                                     throw new RuntimeException(e);
 346                                 }
 347                             }))));
 348             futures.add(configFuture);
 349             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 350                 @Override
 351                 public void onSuccess(Void result) {
 352                     createFibEntries(identifier, update);
 353                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 354                             rd, update.getDestPrefix(), update.getRoutePaths());
 355                 }
 356
 357                 @Override
 358                 public void onFailure(Throwable throwable) {
 359                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 360                             update, throwable);
 361                 }
 362             }, MoreExecutors.directExecutor());
 363             return;
 364         }
 365
 366         //Handle all other routes only on a cluster reboot
 367         if (original.equals(update)) {
 368             //Reboot use-case
 369             createFibEntries(identifier, update);
 370             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 371                     rd, update.getDestPrefix(), update.getRoutePaths());
 372             return;
 373         }
 374
 375         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 376                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 377     }
 378
 379     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 380         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 381         List<SubTransaction> txnObjects =  new ArrayList<>();
 382         final VpnInstanceOpDataEntry vpnInstance =
 383                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 384         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 385         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 386                 + " has null vpnId!");
 387         final Collection<VpnToDpnList> vpnToDpnList;
 388         if (vrfEntry.getParentVpnRd() != null
 389                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 390             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 391             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 392             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 393                 vpnInstance.getVpnToDpnList();
 394             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 395                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 396         } else {
 397             vpnToDpnList = vpnInstance.getVpnToDpnList();
 398         }
 399         final Long vpnId = vpnInstance.getVpnId();
 400         final String rd = vrfTableKey.getRouteDistinguisher();
 401         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 402         if (subnetRoute != null) {
 403             final long elanTag = subnetRoute.getElantag();
 404             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 405                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 406             if (vpnToDpnList != null) {
 407                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 408                     () -> Collections.singletonList(
 409                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 410                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 411                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 412                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 413                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 414                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 415                                 }
 416                             }
 417                         })));
 418             }
 419             return;
 420         }
 421         // Get etherType value based on the IpPrefix address family type
 422         int etherType;
 423         try {
 424             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 425         } catch (IllegalArgumentException ex) {
 426             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 427             return;
 428         }
 429
 430         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 431         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 432             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 433                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 434                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 435                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 436                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 437                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 438                                     try {
 439                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 440                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 441                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 442                                                     TransactionAdapter.toWriteTransaction(tx),
 443                                                     txnObjects);
 444                                         } else {
 445                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 446                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 447                                         }
 448                                     } catch (NullPointerException e) {
 449                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 450                                                 vrfEntry.getDestPrefix(), e);
 451                                     }
 452                                 }
 453                             }
 454                         }
 455                     }
 456                 })), MAX_RETRIES);
 457         }
 458
 459         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 460         if (optVpnUuid.isPresent()) {
 461             String vpnUuid = optVpnUuid.get();
 462             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 463             if (interVpnLink != null) {
 464                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 465                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 466                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 467                         // This is an static route that points to the other endpoint of an InterVpnLink
 468                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 469                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 470                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 471                     }
 472                 });
 473             }
 474         }
 475     }
 476
 477     void refreshFibTables(String rd, String prefix) {
 478         InstanceIdentifier<VrfEntry> vrfEntryId =
 479                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 480                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 481         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 482         if (vrfEntry.isPresent()) {
 483             createFibEntries(vrfEntryId, vrfEntry.get());
 484         }
 485     }
 486
 487     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 488         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 489         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 490         prefixBuilder.setDpnId(lri.getDpnId());
 491         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 492         prefixBuilder.setIpAddress(lri.getPrefix());
 493         // Increment the refCount here
 494         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 495             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 496         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 497         if (!isPresentInList) {
 498             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 499             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 500             vpnInstanceNames.add(vpnInstanceName);
 501             builder.setVpnInstanceList(vpnInstanceNames);
 502             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 503         } else {
 504             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 505         }
 506         return prefixBuilder.build();
 507     }
 508
 509     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 510             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 511         if (tx == null) {
 512             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 513                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 514                 "Error installing subnet route in FIB");
 515             return;
 516         }
 517         int etherType;
 518         try {
 519             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 520         } catch (IllegalArgumentException ex) {
 521             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 522             return;
 523         }
 524         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 525             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 526             synchronized (label.toString().intern()) {
 527                 LabelRouteInfo lri = getLabelRouteInfo(label);
 528                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 529
 530                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 531                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 532                                 fibUtil.getVpnInstanceOpData(rd);
 533                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 534                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 535                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 536                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 537                             }
 538                         }
 539                     }
 540                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 541                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 542                 }
 543             }
 544         });
 545         final List<InstructionInfo> instructions = new ArrayList<>();
 546         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 547             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 548         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 549         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 550         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 551                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 552         if (vrfEntry.getRoutePaths() != null) {
 553             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 554                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 555                     List<ActionInfo> actionsInfos = new ArrayList<>();
 556                     // reinitialize instructions list for LFIB Table
 557                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 558                     actionsInfos.add(new ActionPopMpls(etherType));
 559                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 560                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 561                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 562                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 563
 564                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 565                             NwConstants.ADD_FLOW, tx);
 566                 }
 567             }
 568         }
 569     }
 570
 571     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 572             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 573         List<MatchInfo> matches = new ArrayList<>();
 574
 575         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 576                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 577         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 578         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 579         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 580
 581         InetAddress destPrefix;
 582         try {
 583             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 584         } catch (UnknownHostException e) {
 585             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 586                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 587             return;
 588         }
 589
 590         // Match on VpnId and SubnetBroadCast IP address
 591         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 592         matches.add(MatchEthernetType.IPV4);
 593
 594         if (prefixLength != 0) {
 595             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 596         }
 597
 598         //Action is to drop the packet
 599         List<InstructionInfo> dropInstructions = new ArrayList<>();
 600         List<ActionInfo> actionsInfos = new ArrayList<>();
 601         actionsInfos.add(new ActionDrop());
 602         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 603
 604         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 605         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 606         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 607                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 608
 609         Flow flow = flowEntity.getFlowBuilder().build();
 610         String flowId = flowEntity.getFlowId();
 611         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 612         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 613
 614         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 615                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 616                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 617
 618         if (addOrRemove == NwConstants.ADD_FLOW) {
 619             tx.put(flowInstanceId,flow, true);
 620         } else {
 621             tx.delete(flowInstanceId);
 622         }
 623     }
 624
 625     /*
 626      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 627      * LportDispatcher table (via table 80)
 628      */
 629     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 630                                             final VrfEntry vrfEntry, int etherType) {
 631         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 632         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 633         // packet is commuted from Vpn2 to Vpn1.
 634         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 635         if (!interVpnLink.isActive()) {
 636             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 637                      interVpnLinkName, vrfEntry.getDestPrefix());
 638             return;
 639         }
 640
 641         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 642         if (!optLportTag.isPresent()) {
 643             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 644             return;
 645         }
 646
 647         Long lportTag = optLportTag.get();
 648         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 649         if (label == null) {
 650             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 651                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 652             return;
 653         }
 654         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 655         List<InstructionInfo> instructions = Arrays.asList(
 656             new InstructionApplyActions(actionsInfos),
 657             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 658                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 659                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 660                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 661             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 662         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 663         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 664
 665         for (BigInteger dpId : targetDpns) {
 666             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 667                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 668
 669             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 670                                /*writeTx*/null);
 671         }
 672     }
 673
 674
 675     /*
 676      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 677      */
 678     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 679                                                final VrfEntry vrfEntry, long vpnTag) {
 680         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 681         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 682             && vrfEntry.getRoutePaths().size() == 1);
 683         String destination = vrfEntry.getDestPrefix();
 684         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 685         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 686
 687         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 688         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 689         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 690         if (interVpnLink.getState().or(State.Error) != State.Active) {
 691             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 692                 destination, nextHop, interVpnLinkName);
 693             return;
 694         }
 695
 696         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 697         if (!optOtherEndpointLportTag.isPresent()) {
 698             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 699                 vpnUuid, interVpnLinkName);
 700             return;
 701         }
 702
 703         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 704         if (targetDpns.isEmpty()) {
 705             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 706                 vpnUuid, interVpnLinkName);
 707             return;
 708         }
 709
 710         String[] values = destination.split("/");
 711         String destPrefixIpAddress = values[0];
 712         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 713
 714         List<MatchInfo> matches = new ArrayList<>();
 715         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 716         matches.add(MatchEthernetType.IPV4);
 717
 718         if (prefixLength != 0) {
 719             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 720         }
 721
 722         List<Instruction> instructions =
 723             Arrays.asList(new InstructionWriteMetadata(
 724                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 725                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 726                             .L3VPN_SERVICE_INDEX)),
 727                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 728                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 729
 730         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 731         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 732         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 733             COOKIE_VM_FIB_TABLE, matches, instructions);
 734
 735         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 736             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 737
 738         for (BigInteger dpId : targetDpns) {
 739
 740             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 741                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 742                 dpId, interVpnLink.getInterVpnLinkName());
 743
 744             mdsalManager.installFlow(dpId, flowEntity);
 745         }
 746     }
 747
 748     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 749         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 750         String localNextHopIP = vrfEntry.getDestPrefix();
 751         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 752         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 753         if (localNextHopInfo == null) {
 754             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 755             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 756                     vpnName, usedRds, localNextHopIP);
 757             if (LOG.isDebugEnabled()) {
 758                 LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 759                         vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 760             }
 761             boolean localNextHopSeen = false;
 762             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 763             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 764                 String ipPrefix;
 765                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 766                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 767                 } else {
 768                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 769                 }
 770                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 771                     ipPrefix);
 772                 if (localNextHopInfoLocal != null) {
 773                     localNextHopSeen = true;
 774                     BigInteger dpnId =
 775                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 776                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 777                     returnLocalDpnId.add(dpnId);
 778                 }
 779             }
 780             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 781                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 782                 if (optionalLabel.isPresent()) {
 783                     Long label = optionalLabel.get();
 784                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 785                     synchronized (label.toString().intern()) {
 786                         LabelRouteInfo lri = getLabelRouteInfo(label);
 787                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 788                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 789                                     fibUtil.getVpnInstanceOpData(rd);
 790                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 791                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 792                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 793                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 794                                     localNextHopIP = lri.getPrefix();
 795                                 } else {
 796                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 797                                     localNextHopIP = lri.getPrefix();
 798                                 }
 799                             }
 800                             if (localNextHopInfo != null) {
 801                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 802                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 803                                 if (vpnExtraRoutes.isEmpty()) {
 804                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 805                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 806                                     returnLocalDpnId.add(dpnId);
 807                                 } else {
 808                                     for (Routes extraRoutes : vpnExtraRoutes) {
 809                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 810                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 811                                         returnLocalDpnId.add(dpnId);
 812                                     }
 813                                 }
 814                             }
 815                         }
 816                     }
 817                 }
 818             }
 819             if (returnLocalDpnId.isEmpty()) {
 820                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 821             }
 822         } else {
 823             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 824                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 825             if (dpnId != null) {
 826                 returnLocalDpnId.add(dpnId);
 827             }
 828         }
 829         return returnLocalDpnId;
 830     }
 831
 832     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 833                                                 final Long vpnId, final String rd,
 834                                                 final VrfEntry vrfEntry,
 835                                                 Routes routes, List<Routes> vpnExtraRoutes,
 836                                                 int etherType) {
 837         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 838         if (localNextHopInfo != null) {
 839             long groupId;
 840             long localGroupId;
 841             final BigInteger dpnId = localNextHopInfo.getDpnId();
 842             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 843                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 844                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 845                 return dpnId;
 846             }
 847             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 848                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 849                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 850                 return dpnId;
 851             }
 852             if (!isVpnPresentInDpn(rd, dpnId)) {
 853                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 854                         vpnId, rd, dpnId.toString());
 855                 return BigInteger.ZERO;
 856             }
 857             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 858             String prefix = vrfEntry.getDestPrefix();
 859             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 860             //The loadbalancing group is created only if the extra route has multiple nexthops
 861             //to avoid loadbalancing the discovered routes
 862             if (vpnExtraRoutes != null && routes != null) {
 863                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 864                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 865                 } else {
 866                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 867                 }
 868                 if (vpnExtraRoutes.size() > 1) {
 869                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 870                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 871                 } else if (routes.getNexthopIpList().size() > 1) {
 872                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 873                     localGroupId = groupId;
 874                 } else {
 875                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 876                             prefix, gwMacAddress);
 877                     localGroupId = groupId;
 878                 }
 879             } else {
 880                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 881                         gwMacAddress);
 882                 localGroupId = groupId;
 883             }
 884             if (groupId == FibConstants.INVALID_GROUP_ID) {
 885                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 886                         prefix, rd, interfaceName, dpnId.toString());
 887                 return BigInteger.ZERO;
 888             }
 889             final List<InstructionInfo> instructions = Collections.singletonList(
 890                     new InstructionApplyActions(
 891                             Collections.singletonList(new ActionGroup(groupId))));
 892             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 893                     new InstructionApplyActions(
 894                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 895             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 896             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 897             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 898             jobCoordinator.enqueueJob(jobKey,
 899                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 900                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 901                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 902                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 903                         optLabel.ifPresent(label -> {
 904                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 905                                 LOG.debug(
 906                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 907                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 908                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 909                                         nextHopAddressList);
 910                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 911                                         NwConstants.ADD_FLOW, tx);
 912                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 913                             } else {
 914                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 915                                                 + "route. LFib and Terminating table entries will not be created.",
 916                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 917                             }
 918                         });
 919                     }
 920                 })));
 921             return dpnId;
 922         }
 923         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 924                 vpnName);
 925         return BigInteger.ZERO;
 926     }
 927
 928     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 929         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 930         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 931         if (dpnInVpn.isPresent()) {
 932             return true;
 933         }
 934         return false;
 935     }
 936
 937     private LabelRouteInfo getLabelRouteInfo(Long label) {
 938         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 939             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 940         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 941         if (opResult.isPresent()) {
 942             return opResult.get();
 943         }
 944         return null;
 945     }
 946
 947     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 948             TypedWriteTransaction<Operational> tx) {
 949         if (lri == null) {
 950             return true;
 951         }
 952
 953         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 954         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 955             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 956
 957         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 958             ? lri.getVpnInstanceList() : new ArrayList<>();
 959         if (vpnInstancesList.contains(vpnInstanceName)) {
 960             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 961             vpnInstancesList.remove(vpnInstanceName);
 962         }
 963         if (vpnInstancesList.isEmpty()) {
 964             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 965             if (tx != null) {
 966                 tx.delete(lriId);
 967             } else {
 968                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 969             }
 970             return true;
 971         } else {
 972             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 973             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 974             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 975         }
 976         return false;
 977     }
 978
 979     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 980                                       TypedWriteTransaction<Configuration> tx) {
 981         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 982
 983         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 984
 985         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 986             dpId, label, groupId);
 987     }
 988
 989     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 990                                                 TypedWriteTransaction<Configuration> tx) {
 991         List<MatchInfo> mkMatches = new ArrayList<>();
 992
 993         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 994             destDpId, label, actionsInfos);
 995
 996         // Matching metadata
 997         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 998         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 999
1000         List<InstructionInfo> mkInstructions = new ArrayList<>();
1001         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1002
1003         FlowEntity terminatingServiceTableFlowEntity =
1004             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1005             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1006                 String.format("%s:%d", "TST Flow Entry ", label),
1007             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1008
1009         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1010
1011         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1012
1013         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1014         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1015             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1016             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1017             .child(Flow.class, flowKey).build();
1018         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1019     }
1020
1021     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1022         FlowEntity flowEntity;
1023         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1024         List<MatchInfo> mkMatches = new ArrayList<>();
1025         // Matching metadata
1026         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1027         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1028             NwConstants.INTERNAL_TUNNEL_TABLE,
1029             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1030             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1031             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1032         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1033         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1034         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1035             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1036             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1037
1038         tx.delete(flowInstanceId);
1039         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1040     }
1041
1042     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1043         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1044         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1045         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1046         boolean shouldUpdateNonEcmpLocalNextHop = true;
1047         if (localNextHopInfo == null) {
1048             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1049             if (usedRds.size() > 1) {
1050                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1051                         vrfEntry.getDestPrefix(), vpnName, rd);
1052                 return returnLocalDpnId;
1053             }
1054             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1055             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1056             //in the vpn
1057             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1058                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1059             if (extraRouteOptional.isPresent()) {
1060                 Routes extraRoute = extraRouteOptional.get();
1061                 String ipPrefix;
1062                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1063                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1064                 } else {
1065                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1066                 }
1067                 if (extraRoute.getNexthopIpList().size() > 1) {
1068                     shouldUpdateNonEcmpLocalNextHop = false;
1069                 }
1070                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1071                 if (localNextHopInfo != null) {
1072                     String localNextHopIP = localNextHopInfo.getIpAddress();
1073                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1074                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1075                     if (!dpnId.equals(BigInteger.ZERO)) {
1076                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1077                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1078                         returnLocalDpnId.add(dpnId);
1079                     }
1080                 } else {
1081                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1082                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1083                 }
1084             }
1085
1086             if (localNextHopInfo == null) {
1087                 /* Imported VRF entry */
1088                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1089                 if (optionalLabel.isPresent()) {
1090                     Long label = optionalLabel.get();
1091                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1092                     LabelRouteInfo lri = getLabelRouteInfo(label);
1093                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1094                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1095                         prefixBuilder.setDpnId(lri.getDpnId());
1096                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1097                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1098                         if (!dpnId.equals(BigInteger.ZERO)) {
1099                             returnLocalDpnId.add(dpnId);
1100                         }
1101                     }
1102                 }
1103             }
1104
1105         } else {
1106             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1107             String localNextHopIP = localNextHopInfo.getIpAddress();
1108             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1109                     shouldUpdateNonEcmpLocalNextHop);
1110             if (!dpnId.equals(BigInteger.ZERO)) {
1111                 returnLocalDpnId.add(dpnId);
1112             }
1113         }
1114
1115         return returnLocalDpnId;
1116     }
1117
1118     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1119             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1120             boolean shouldUpdateNonEcmpLocalNextHop) {
1121         if (localNextHopInfo != null) {
1122             final BigInteger dpnId = localNextHopInfo.getDpnId();
1123             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1124                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1125                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1126                 return dpnId;
1127             }
1128             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1129                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1130                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1131                 return dpnId;
1132             }
1133
1134             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1135                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1136                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1137                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1138                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1139                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1140                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1141                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1142                                         NwConstants.DEL_FLOW, tx);
1143                                 removeTunnelTableEntry(dpnId, label, tx);
1144                             });
1145                         }
1146                     }
1147                 })));
1148             //TODO: verify below adjacency call need to be optimized (?)
1149             //In case of the removal of the extra route, the loadbalancing group is updated
1150             if (shouldUpdateNonEcmpLocalNextHop) {
1151                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1152             }
1153             return dpnId;
1154         }
1155         return BigInteger.ZERO;
1156     }
1157
1158     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1159             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1160         if (tx == null) {
1161             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1162                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1163                 "Error creating remote FIB entry");
1164             return;
1165         }
1166
1167         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1168         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1169                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1170
1171         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1172         if (adjacencyResults.isEmpty()) {
1173             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1174                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1175             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1176             return;
1177         }
1178
1179         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1180         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1181                 vpnName, usedRds, vrfEntry.getDestPrefix());
1182         // create loadbalancing groups for extra routes only when the extra route is present behind
1183         // multiple VMs
1184         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1185                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1186             List<InstructionInfo> instructions = new ArrayList<>();
1187             // Obtain the local routes for this particular dpn.
1188             java.util.Optional<Routes> routes = vpnExtraRoutes
1189                     .stream()
1190                     .filter(route -> {
1191                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1192                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1193                         if (prefixToInterface == null) {
1194                             return false;
1195                         }
1196                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1197                     }).findFirst();
1198             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1199                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1200             if (groupId == FibConstants.INVALID_GROUP_ID) {
1201                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1202                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1203                 return;
1204             }
1205             List<ActionInfo> actionInfos =
1206                     Collections.singletonList(new ActionGroup(groupId));
1207             instructions.add(new InstructionApplyActions(actionInfos));
1208             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1209                     NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1210         } else {
1211             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry,
1212                 TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null);
1213         }
1214
1215         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1216     }
1217
1218     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1219     /* Get interface info from prefix to interface mapping;
1220         Use the interface info to get the corresponding vpn interface op DS entry,
1221         remove the adjacency corresponding to this fib entry.
1222         If adjacency removed is the last adjacency, clean up the following:
1223          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1224          - prefix to interface entry
1225          - vpn interface op DS
1226      */
1227         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1228         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1229         if (prefixInfo == null) {
1230             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1231             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1232             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1233             if (extraRoute != null) {
1234                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1235                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1236                     if (nextHopIp != null) {
1237                         String ipPrefix;
1238                         if (isIpv4Address(nextHopIp)) {
1239                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1240                         } else {
1241                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1242                         }
1243                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1244                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1245                     }
1246                 }
1247             }
1248             if (prefixInfo == null) {
1249                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1250                 if (optionalLabel.isPresent()) {
1251                     Long label = optionalLabel.get();
1252                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1253                     LabelRouteInfo lri = getLabelRouteInfo(label);
1254                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1255                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1256                         prefixBuilder.setDpnId(lri.getDpnId());
1257                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1258                         prefixBuilder.setIpAddress(lri.getPrefix());
1259                         prefixInfo = prefixBuilder.build();
1260                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1261                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1262                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1263                     }
1264                 }
1265             }
1266         } else {
1267             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1268         }
1269     }
1270
1271     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1272                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1273
1274         if (prefixInfo == null) {
1275             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1276                     vrfEntry.getDestPrefix(), vpnId, rd);
1277             return; //Don't have any info for this prefix (shouldn't happen); need to return
1278         }
1279
1280         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1281             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1282                     vrfEntry.getDestPrefix(), vpnId, rd);
1283             return;
1284         }
1285
1286         String ifName = prefixInfo.getVpnInterfaceName();
1287         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1288             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1289     }
1290
1291     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1292         Prefixes prefixInfo;
1293         Long vpnId;
1294         String rd;
1295         VrfEntry vrfEntry;
1296         Routes extraRoute;
1297
1298         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1299                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1300             this.prefixInfo = prefixInfo;
1301             this.vpnId = vpnId;
1302             this.rd = rd;
1303             this.vrfEntry = vrfEntry;
1304             this.extraRoute = extraRoute;
1305         }
1306
1307         @Override
1308         public List<ListenableFuture<Void>> call() {
1309             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1310             // to call the respective helpers.
1311             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1312                 //First Cleanup LabelRouteInfo
1313                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1314                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1315                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1316                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1317                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1318                         synchronized (label.toString().intern()) {
1319                             LabelRouteInfo lri = getLabelRouteInfo(label);
1320                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1321                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1322                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1323                                         fibUtil.getVpnInstanceOpData(rd);
1324                                 String vpnInstanceName = "";
1325                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1326                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1327                                 }
1328                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1329                                 if (lriRemoved) {
1330                                     String parentRd = lri.getParentVpnRd();
1331                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1332                                             parentRd, vrfEntry.getDestPrefix()));
1333                                 }
1334                             } else {
1335                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1336                                         rd, vrfEntry.getDestPrefix()));
1337                             }
1338                         }
1339                     });
1340                 }
1341                 String ifName = prefixInfo.getVpnInterfaceName();
1342                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1343                 String vpnName = null;
1344
1345                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1346                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1347                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1348                     if (vpnNameOpt.isPresent()) {
1349                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1350                     }
1351                 }
1352                 if (optVpnName.isPresent()) {
1353                     vpnName = optVpnName.get();
1354                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1355                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1356                     if (opVpnInterface.isPresent()) {
1357                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1358                         if (vpnId != associatedVpnId) {
1359                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1360                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1361                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1362                             return;
1363                         } else {
1364                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1365                                     vrfEntry.getDestPrefix(), associatedVpnId);
1366                         }
1367                     }
1368                 }
1369                 if (extraRoute != null) {
1370                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1371                     //Only one used Rd present in case of removal event
1372                     String usedRd = usedRds.get(0);
1373                     if (optVpnName.isPresent()) {
1374                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1375                                         vrfEntry.getDestPrefix()));
1376                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1377                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1378                         nextHopManager.removeNextHopPointer(nextHopManager
1379                                 .getRemoteSelectGroupKey(vpnId, vrfEntry.getDestPrefix()));
1380                         nextHopManager.removeNextHopPointer(nextHopManager
1381                                 .getLocalSelectGroupKey(vpnId, vrfEntry.getDestPrefix()));
1382                     }
1383                 }
1384                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1385             }));
1386         }
1387     }
1388
1389     /**
1390      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1391      * Remove Adjacency from VPNInterfaceOpData.
1392      * if Adjacency != primary.
1393      * if Adjacency == primary , then mark it for deletion.
1394      * Remove entire VPNinterfaceOpData Entry.
1395      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1396      * @param vrfEntry - VrfEntry removed
1397      * @param ifName - Interface name from VRFentry
1398      * @param vpnName - VPN name of corresponding VRF
1399      * @param tx - ReadWrite Tx
1400      */
1401     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1402                                                           TypedReadWriteTransaction<Operational> tx)
1403             throws ExecutionException, InterruptedException {
1404         InstanceIdentifier<Adjacency> adjacencyIid =
1405                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1406         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1407         if (adjacencyOptional.isPresent()) {
1408             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1409                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1410             } else {
1411                 tx.merge(adjacencyIid,
1412                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1413             }
1414         }
1415
1416         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1417
1418         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1419             return;
1420         }
1421
1422         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1423                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1424                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1425                         && adjacency.isMarkedForDeletion() != null
1426                         && adjacency.isMarkedForDeletion()
1427         )) {
1428             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1429             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1430         }
1431     }
1432
1433     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1434         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1435         final String rd = vrfTableKey.getRouteDistinguisher();
1436         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1437         if (vpnInstance == null) {
1438             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1439             return;
1440         }
1441         final Collection<VpnToDpnList> vpnToDpnList;
1442         if (vrfEntry.getParentVpnRd() != null
1443                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1444             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1445             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1446             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1447                     vpnInstance.getVpnToDpnList();
1448             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1449                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1450         } else {
1451             vpnToDpnList = vpnInstance.getVpnToDpnList();
1452         }
1453
1454         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1455         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1456         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1457         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1458         if (subnetRoute != null) {
1459             long elanTag = subnetRoute.getElantag();
1460             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1461                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1462             if (vpnToDpnList != null) {
1463                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1464                     () -> Collections.singletonList(
1465                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1466                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1467
1468                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1469                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1470                                     TransactionAdapter.toWriteTransaction(tx), null);
1471                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1472                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1473                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1474                                 }
1475
1476                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1477                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1478                             }
1479                         })));
1480             }
1481             optionalLabel.ifPresent(label -> {
1482                 synchronized (label.toString().intern()) {
1483                     LabelRouteInfo lri = getLabelRouteInfo(label);
1484                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1485                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1486                                 fibUtil.getVpnInstanceOpData(rd);
1487                         String vpnInstanceName = "";
1488                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1489                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1490                         }
1491                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1492                         if (lriRemoved) {
1493                             String parentRd = lri.getParentVpnRd();
1494                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1495                                     parentRd, vrfEntry.getDestPrefix()));
1496                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1497                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1498                         }
1499                     } else {
1500                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1501                                 rd, vrfEntry.getDestPrefix()));
1502                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1503                                 label, rd, vrfEntry.getDestPrefix());
1504                     }
1505                 }
1506             });
1507             return;
1508         }
1509
1510         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1511             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1512         if (vpnToDpnList != null) {
1513             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1514                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1515             String jobKey;
1516             Optional<Routes> extraRouteOptional;
1517             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1518             if (usedRds != null && !usedRds.isEmpty()) {
1519                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1520                 if (usedRds.size() > 1) {
1521                     LOG.error("The extra route prefix is still present in some DPNs");
1522                     return ;
1523                 } else {
1524                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1525                     //is not present in any other DPN
1526                     extraRouteOptional = VpnExtraRouteHelper
1527                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1528                 }
1529             } else {
1530                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1531                 extraRouteOptional = Optional.absent();
1532             }
1533
1534             jobCoordinator.enqueueJob(jobKey,
1535                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1536                     if (localDpnIdList.size() <= 0) {
1537                         for (VpnToDpnList curDpn : vpnToDpnList) {
1538                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1539                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1540                                 TransactionAdapter.toWriteTransaction(tx));
1541                         }
1542                     } else {
1543                         for (BigInteger localDpnId : localDpnIdList) {
1544                             for (VpnToDpnList curDpn : vpnToDpnList) {
1545                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1546                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1547                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1548                                         TransactionAdapter.toWriteTransaction(tx));
1549                                 }
1550                             }
1551                         }
1552                     }
1553                 })), MAX_RETRIES);
1554         }
1555
1556         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1557         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1558         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1559
1560         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1561         // of the interVpnLink.
1562         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1563         if (optVpnUuid.isPresent()) {
1564             String vpnUuid = optVpnUuid.get();
1565             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1566                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1567                 if (optInterVpnLink.isPresent()) {
1568                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1569                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1570                         // This is route that points to the other endpoint of an InterVpnLink
1571                         // In that case, we should look for the FIB table pointing to
1572                         // LPortDispatcher table and remove it.
1573                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1574                     }
1575                 }
1576             });
1577         }
1578
1579     }
1580
1581     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1582                                     int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1583         if (tx == null) {
1584             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1585                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1586                 "Error making LFIB table entry");
1587             return;
1588         }
1589
1590         List<MatchInfo> matches = new ArrayList<>();
1591         matches.add(MatchEthernetType.MPLS_UNICAST);
1592         matches.add(new MatchMplsLabel(label));
1593
1594         // Install the flow entry in L3_LFIB_TABLE
1595         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1596
1597         FlowEntity flowEntity;
1598         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1599             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1600         Flow flow = flowEntity.getFlowBuilder().build();
1601         String flowId = flowEntity.getFlowId();
1602         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1603         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1604         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1605             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1606             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1607
1608         if (addOrRemove == NwConstants.ADD_FLOW) {
1609             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1610         } else {
1611             tx.delete(flowInstanceId);
1612         }
1613
1614         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1615             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1616     }
1617
1618     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1619                                     final FutureCallback<List<Void>> callback) {
1620         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1621         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1622             () -> {
1623                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1624                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1625                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1626                         LogicalDatastoreType.CONFIGURATION, id);
1627                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1628                 if (!vrfTable.isPresent()) {
1629                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1630                     if (callback != null) {
1631                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1632                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1633                     }
1634                     return futures;
1635                 }
1636                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1637                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1638                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1639                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1640                             if (subnetRoute != null) {
1641                                 long elanTag = subnetRoute.getElantag();
1642                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1643                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1644                                         tx);
1645                                 continue;
1646                             }
1647                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1648                             if (routerInt != null) {
1649                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1650                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1651                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1652                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1653                                         NwConstants.ADD_FLOW);
1654                                 continue;
1655                             }
1656                             //Handle local flow creation for imports
1657                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1658                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1659                                 if (optionalLabel.isPresent()) {
1660                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1661                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1662                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1663                                         if (lri.getDpnId().equals(dpnId)) {
1664                                             try {
1665                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1666                                                         vrfEntry.getDestPrefix());
1667                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1668                                             } catch (IllegalArgumentException ex) {
1669                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1670                                                         vrfEntry.getDestPrefix());
1671                                             }
1672                                             continue;
1673                                         }
1674                                     }
1675                                 }
1676                             }
1677                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1678                                     vrfEntry, dpnId);
1679                             if (shouldCreateRemoteFibEntry) {
1680                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1681                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1682                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1683                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1684                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1685                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1686                                 } else {
1687                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1688                                             vrfEntry, tx);
1689                                 }
1690                             }
1691                         }
1692                     }));
1693                     if (callback != null) {
1694                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1695                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1696                     }
1697                 }
1698                 return futures;
1699             });
1700     }
1701
1702     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1703                                             final String localNextHopIp, final String remoteNextHopIp) {
1704         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1705             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1706         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1707         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1708         List<SubTransaction> txnObjects =  new ArrayList<>();
1709         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1710         if (vrfTable.isPresent()) {
1711             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1712                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1713                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1714                         vrfTable.get().getVrfEntry().stream()
1715                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1716                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1717                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1718                     }
1719                 })));
1720         }
1721     }
1722
1723     public void manageRemoteRouteOnDPN(final boolean action,
1724                                        final BigInteger localDpnId,
1725                                        final long vpnId,
1726                                        final String rd,
1727                                        final String destPrefix,
1728                                        final String destTepIp,
1729                                        final long label) {
1730         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1731
1732         if (vpnInstance == null) {
1733             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1734             return;
1735         }
1736
1737         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1738             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1739                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1740                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1741                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1742                     if (vrfEntry == null) {
1743                         return;
1744                     }
1745                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1746                             action, localDpnId, vpnId, rd, destPrefix);
1747                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1748                     VrfEntry modVrfEntry;
1749                     if (routePathList == null || routePathList.isEmpty()) {
1750                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1751                                 Collections.singletonList(destTepIp),
1752                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1753                     } else {
1754                         modVrfEntry = vrfEntry;
1755                     }
1756
1757                     if (action) {
1758                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1759                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1760                                 modVrfEntry, tx);
1761                     } else {
1762                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1763                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1764                                 vrfEntry.getDestPrefix());
1765                         if (usedRds.size() > 1) {
1766                             LOG.debug("The extra route prefix is still present in some DPNs");
1767                             return;
1768                         }
1769                         //Is this fib route an extra route? If yes, get the nexthop which would be
1770                         //an adjacency in the vpn
1771                         Optional<Routes> extraRouteOptional = Optional.absent();
1772                         if (usedRds.size() != 0) {
1773                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1774                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1775                                     usedRds.get(0), vrfEntry.getDestPrefix());
1776                         }
1777                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1778                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1779                     }
1780                 }
1781             })));
1782     }
1783
1784     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1785                                  final FutureCallback<List<Void>> callback) {
1786         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1787         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1788             () -> {
1789                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1790                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1791                 List<SubTransaction> txnObjects =  new ArrayList<>();
1792                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1793                         LogicalDatastoreType.CONFIGURATION, id);
1794                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1795                 if (vrfTable.isPresent()) {
1796                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1797                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1798                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1799                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1800                                 /* Handle subnet routes here */
1801                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1802                                 if (subnetRoute != null && !fibUtil
1803                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1804                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1805                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1806                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1807                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1808                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1809                                     if (routePaths != null) {
1810                                         for (RoutePaths routePath : routePaths) {
1811                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1812                                                     DEFAULT_FIB_FLOW_PRIORITY,
1813                                                     NwConstants.DEL_FLOW, tx);
1814                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1815                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1816                                                     vrfEntry.getDestPrefix());
1817                                         }
1818                                     }
1819                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1820                                             NwConstants.DEL_FLOW, tx);
1821                                     continue;
1822                                 }
1823                                 // ping responder for router interfaces
1824                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1825                                 if (routerInt != null) {
1826                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1827                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1828                                             routerInt.getMacAddress());
1829                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1830                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1831                                             NwConstants.DEL_FLOW);
1832                                     continue;
1833                                 }
1834
1835                                 //Handle local flow deletion for imports
1836                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1837                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1838                                     if (optionalLabel.isPresent()) {
1839                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1840                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1841                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1842                                                 lri) && lri.getDpnId().equals(dpnId)) {
1843                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1844                                         }
1845                                     }
1846                                 }
1847
1848                                 // Passing null as we don't know the dpn
1849                                 // to which prefix is attached at this point
1850                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1851                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1852                                 Optional<Routes> extraRouteOptional;
1853                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1854                                 //an adjacency in the vpn
1855                                 if (usedRds != null && !usedRds.isEmpty()) {
1856                                     if (usedRds.size() > 1) {
1857                                         LOG.error("The extra route prefix is still present in some DPNs");
1858                                         return;
1859                                     } else {
1860                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1861                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1862
1863                                     }
1864                                 } else {
1865                                     extraRouteOptional = Optional.absent();
1866                                 }
1867                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1868                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1869                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1870                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1871                                 } else {
1872                                     if (subnetRoute == null || !fibUtil
1873                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1874                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1875                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1876                                             TransactionAdapter.toWriteTransaction(tx));
1877                                     }
1878                                 }
1879                             }
1880                         }));
1881                     }
1882                     if (callback != null) {
1883                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1884                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1885                     }
1886                 } else {
1887                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1888                 }
1889                 return futures;
1890             });
1891
1892     }
1893
1894     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1895                                            final String localNextHopIp, final String remoteNextHopIp) {
1896         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1897                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1898             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1899         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1900         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1901         List<SubTransaction> txnObjects =  new ArrayList<>();
1902         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1903         if (vrfTable.isPresent()) {
1904             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1905                 () -> {
1906                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1907                         return Collections.singletonList(
1908                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1909                                 tx -> vrfTable.get().getVrfEntry().stream()
1910                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1911                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1912                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1913                                         txnObjects))));
1914                     }
1915                 });
1916         }
1917     }
1918
1919     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1920         InstanceIdentifierBuilder<VrfTables> idBuilder =
1921             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1922         return idBuilder.build();
1923     }
1924
1925     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1926         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1927                 .FLOWID_SEPARATOR + nextHop;
1928     }
1929
1930     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1931         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1932                 + tableMiss + FLOWID_PREFIX;
1933     }
1934
1935     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1936         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1937             .child(VrfTables.class, new VrfTablesKey(rd))
1938             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1939         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1940         if (vrfEntry.isPresent()) {
1941             return vrfEntry.get();
1942         }
1943         return null;
1944     }
1945
1946     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1947                                              final String vpnName,
1948                                              final VrfEntry vrfEntry) {
1949         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1950
1951         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1952         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1953
1954         if (targetDpns.isEmpty()) {
1955             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1956             return;
1957         }
1958
1959         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1960         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1961
1962         // delete from FIB
1963         //
1964         optNextHop.ifPresent(nextHop -> {
1965             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1966             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1967             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1968                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1969
1970             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1971             for (BigInteger dpId : targetDpns) {
1972                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1973                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1974
1975                 mdsalManager.removeFlow(dpId, flow);
1976             }
1977         });
1978
1979         // delete from LFIB
1980         //
1981         optLabel.ifPresent(label -> {
1982             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1983
1984             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1985                 for (BigInteger dpId : targetDpns) {
1986                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1987                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1988                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1989                             tx);
1990                 }
1991             }), LOG, "Error removing flows");
1992         });
1993     }
1994
1995     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1996             List<String> nextHopAddressList, LabelRouteInfo lri) {
1997         return lri != null && lri.getPrefix().equals(prefix)
1998                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1999     }
2000
2001     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2002         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2003             return true;
2004         }
2005
2006         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2007         if (prefix != null) {
2008             BigInteger prefixDpnId = prefix.getDpnId();
2009             if (dpnId.equals(prefixDpnId)) {
2010                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2011                         vrfEntry, dpnId);
2012                 return false;
2013             }
2014         }
2015         return true;
2016     }
2017 }