fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  34 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  35 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  37 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  38 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  39 import org.opendaylight.genius.mdsalutil.ActionInfo;
  40 import org.opendaylight.genius.mdsalutil.FlowEntity;
  41 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.MatchInfo;
  44 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  49 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  50 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  51 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  52 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  53 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  54 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  55 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  56 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  58 import org.opendaylight.genius.utils.ServiceIndex;
  59 import org.opendaylight.genius.utils.batching.SubTransaction;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  62 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  63 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  64 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  65 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  66 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  67 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  68 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  69 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 101 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 102 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 103 import org.slf4j.Logger;
 104 import org.slf4j.LoggerFactory;
 105
 106
 107 @Singleton
 108 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 109
 110     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 111     private static final String FLOWID_PREFIX = "L3.";
 112     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 113     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 114     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 115     private static final int LFIB_INTERVPN_PRIORITY = 15;
 116     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 117     private static final int MAX_RETRIES = 3;
 118     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 123     private final IMdsalApiManager mdsalManager;
 124     private final NexthopManager nextHopManager;
 125     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 126     private final BaseVrfEntryHandler baseVrfEntryHandler;
 127     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 128     private final JobCoordinator jobCoordinator;
 129     private final IElanService elanManager;
 130     private final FibUtil fibUtil;
 131     private final InterVpnLinkCache interVpnLinkCache;
 132     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 133
 134     @Inject
 135     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 136                             final NexthopManager nexthopManager,
 137                             final IElanService elanManager,
 138                             final BaseVrfEntryHandler vrfEntryHandler,
 139                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 140                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 141                             final JobCoordinator jobCoordinator,
 142                             final FibUtil fibUtil,
 143                             final InterVpnLinkCache interVpnLinkCache) {
 144         super(VrfEntry.class, VrfEntryListener.class);
 145         this.dataBroker = dataBroker;
 146         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 147         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 148         this.mdsalManager = mdsalApiManager;
 149         this.nextHopManager = nexthopManager;
 150         this.elanManager = elanManager;
 151         this.baseVrfEntryHandler = vrfEntryHandler;
 152         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 153         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 154         this.jobCoordinator = jobCoordinator;
 155         this.fibUtil = fibUtil;
 156         this.interVpnLinkCache = interVpnLinkCache;
 157     }
 158
 159     @Override
 160     @PostConstruct
 161     public void init() {
 162         LOG.info("{} init", getClass().getSimpleName());
 163         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 164     }
 165
 166     @Override
 167     @SuppressWarnings("checkstyle:IllegalCatch")
 168     public void close() {
 169         closeables.forEach(c -> {
 170             try {
 171                 c.close();
 172             } catch (Exception e) {
 173                 LOG.warn("Error closing {}", c, e);
 174             }
 175         });
 176     }
 177
 178     @Override
 179     protected VrfEntryListener getDataTreeChangeListener() {
 180         return VrfEntryListener.this;
 181     }
 182
 183     @Override
 184     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 185         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 186     }
 187
 188     @Override
 189     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 190         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 191         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 192         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 193                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 194         addFibEntries(identifier, vrfEntry, rd);
 195         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 196                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 197     }
 198
 199     //This method is temporary. Eventually Factory design pattern will be used to get
 200     // right VrfEntryhandle and invoke its methods.
 201     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 202         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 203             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 204             return;
 205         }
 206         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 207             LOG.info("EVPN flows need to be programmed.");
 208             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 209                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 210             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 211             closeables.add(evpnVrfEntryHandler);
 212             return;
 213         }
 214         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
 215         if (routerInt != null) {
 216             // ping responder for router interfaces
 217             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 218             return;
 219         }
 220         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 221             createFibEntries(identifier, vrfEntry);
 222             return;
 223         }
 224     }
 225
 226     @Override
 227     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 228         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 229         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 230         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 231                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 232         removeFibEntries(identifier, vrfEntry, rd);
 233         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 234             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 235     }
 236
 237     //This method is temporary. Eventually Factory design pattern will be used to get
 238     // right VrfEntryhandle and invoke its methods.
 239     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 240         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 241             LOG.info("EVPN flows to be deleted");
 242             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 243                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 244             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 245             closeables.add(evpnVrfEntryHandler);
 246             return;
 247         }
 248         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
 249         if (routerInt != null) {
 250             // ping responder for router interfaces
 251             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 252             return;
 253         }
 254         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 255             deleteFibEntries(identifier, vrfEntry);
 256             return;
 257         }
 258         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 259             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 260             return;
 261         }
 262     }
 263
 264     @Override
 265     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 266     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 267     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 268     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 269         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 270         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 271         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 272                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 273         // Handle BGP Routes first
 274         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 275             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 276             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 277                     rd, update.getDestPrefix(), update.getRoutePaths());
 278             return;
 279         }
 280
 281         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 282             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 283             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 284             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 285
 286             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 287             //updated vrf entry route path is null. This can happen during tunnel events.
 288             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 289             List<String> usedRds = new ArrayList<>();
 290             if (optVpnInstance.isPresent()) {
 291                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 292                         update.getDestPrefix());
 293             }
 294             // If original VRF Entry had nexthop null , but update VRF Entry
 295             // has nexthop , route needs to be created on remote Dpns
 296             if (originalRoutePath == null || originalRoutePath.isEmpty()
 297                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 298                 // TODO(vivek): Though ugly, Not handling this code now, as each
 299                 // tep add event will invoke flow addition
 300                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 301                         + " This event is IGNORED here.", update.getDestPrefix());
 302                 return;
 303             }
 304
 305             // If original VRF Entry had valid nexthop , but update VRF Entry
 306             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 307             if (updateRoutePath == null || updateRoutePath.isEmpty()
 308                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 309                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 310                         + "This event is IGNORED here.", update.getDestPrefix());
 311                 return;
 312             }
 313             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 314             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 315             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 316             ListenableFuture<Void> future =
 317                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> nextHopsRemoved.parallelStream()
 318                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 319                                     tx, nextHopRemoved, rd, update.getDestPrefix())));
 320             Futures.addCallback(future, new FutureCallback<Void>() {
 321                 @Override
 322                 public void onSuccess(Void result) {
 323                     createFibEntries(identifier, update);
 324                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 325                             rd, update.getDestPrefix(), update.getRoutePaths());
 326                 }
 327
 328                 @Override
 329                 public void onFailure(Throwable throwable) {
 330                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 331                             update, throwable);
 332                 }
 333             }, MoreExecutors.directExecutor());
 334         }
 335
 336         //Handle all other routes only on a cluster reboot
 337         if (original.equals(update)) {
 338             //Reboot use-case
 339             createFibEntries(identifier, update);
 340             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 341                     rd, update.getDestPrefix(), update.getRoutePaths());
 342             return;
 343         }
 344
 345         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 346                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 347     }
 348
 349     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 350         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 351         List<SubTransaction> txnObjects =  new ArrayList<>();
 352         final VpnInstanceOpDataEntry vpnInstance =
 353                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 354         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 355         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 356                 + " has null vpnId!");
 357         final Collection<VpnToDpnList> vpnToDpnList;
 358         if (vrfEntry.getParentVpnRd() != null
 359                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 360             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 361             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 362             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 363                 vpnInstance.getVpnToDpnList();
 364             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 365                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 366         } else {
 367             vpnToDpnList = vpnInstance.getVpnToDpnList();
 368         }
 369         final Long vpnId = vpnInstance.getVpnId();
 370         final String rd = vrfTableKey.getRouteDistinguisher();
 371         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 372         if (subnetRoute != null) {
 373             final long elanTag = subnetRoute.getElantag();
 374             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 375                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 376             if (vpnToDpnList != null) {
 377                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 378                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 379                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 380                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 381                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 382                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 383                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 384                             }
 385                         }
 386                     })));
 387             }
 388             return;
 389         }
 390
 391         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 392         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 393             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 394                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 395                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 396                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 397                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 398                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 399                                     try {
 400                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 401                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 402                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 403                                                     txnObjects);
 404                                         } else {
 405                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 406                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 407                                         }
 408                                     } catch (NullPointerException e) {
 409                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 410                                                 vrfEntry.getDestPrefix(), e);
 411                                     }
 412                                 }
 413                             }
 414                         }
 415                     }
 416                 })), MAX_RETRIES);
 417         }
 418
 419         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 420         if (optVpnUuid.isPresent()) {
 421             String vpnUuid = optVpnUuid.get();
 422             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 423             if (interVpnLink != null) {
 424                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 425                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 426                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 427                         // This is an static route that points to the other endpoint of an InterVpnLink
 428                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 429                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 430                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
 431                     }
 432                 });
 433             }
 434         }
 435     }
 436
 437     void refreshFibTables(String rd, String prefix) {
 438         InstanceIdentifier<VrfEntry> vrfEntryId =
 439                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 440                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 441         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 442         if (vrfEntry.isPresent()) {
 443             createFibEntries(vrfEntryId, vrfEntry.get());
 444         }
 445     }
 446
 447     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 448         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 449         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 450         prefixBuilder.setDpnId(lri.getDpnId());
 451         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 452         prefixBuilder.setIpAddress(lri.getPrefix());
 453         // Increment the refCount here
 454         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 455             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 456         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 457         if (!isPresentInList) {
 458             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 459             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 460             vpnInstanceNames.add(vpnInstanceName);
 461             builder.setVpnInstanceList(vpnInstanceNames);
 462             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 463         } else {
 464             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 465         }
 466         return prefixBuilder.build();
 467     }
 468
 469     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 470                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 471         if (tx == null) {
 472             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 473                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 474                 "Error installing subnet route in FIB");
 475             return;
 476         }
 477         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 478             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 479             synchronized (label.toString().intern()) {
 480                 LabelRouteInfo lri = getLabelRouteInfo(label);
 481                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 482
 483                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 484                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 485                                 fibUtil.getVpnInstanceOpData(rd);
 486                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 487                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 488                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 489                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 490                             }
 491                         }
 492                     }
 493                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 494                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 495                 }
 496             }
 497         });
 498         final List<InstructionInfo> instructions = new ArrayList<>();
 499         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 500             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 501         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 502         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 503         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 504                 NwConstants.ADD_FLOW, tx, null);
 505
 506         if (vrfEntry.getRoutePaths() != null) {
 507             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 508                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 509                     List<ActionInfo> actionsInfos = new ArrayList<>();
 510                     // reinitialize instructions list for LFIB Table
 511                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 512
 513                     actionsInfos.add(new ActionPopMpls());
 514                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 515                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 516                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 517                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 518
 519                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 520                             NwConstants.ADD_FLOW, tx);
 521                 }
 522             }
 523         }
 524     }
 525
 526     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 527                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 528         List<MatchInfo> matches = new ArrayList<>();
 529
 530         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 531                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 532         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 533         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 534         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 535
 536         InetAddress destPrefix;
 537         try {
 538             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 539         } catch (UnknownHostException e) {
 540             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 541                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 542             return;
 543         }
 544
 545         // Match on VpnId and SubnetBroadCast IP address
 546         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 547         matches.add(MatchEthernetType.IPV4);
 548
 549         if (prefixLength != 0) {
 550             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 551         }
 552
 553         //Action is to drop the packet
 554         List<InstructionInfo> dropInstructions = new ArrayList<>();
 555         List<ActionInfo> actionsInfos = new ArrayList<>();
 556         actionsInfos.add(new ActionDrop());
 557         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 558
 559         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 560         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 561         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 562                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 563
 564         Flow flow = flowEntity.getFlowBuilder().build();
 565         String flowId = flowEntity.getFlowId();
 566         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 567         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 568
 569         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 570                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 571                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 572
 573         if (addOrRemove == NwConstants.ADD_FLOW) {
 574             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 575         } else {
 576             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 577         }
 578     }
 579
 580     /*
 581      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 582      * LportDispatcher table (via table 80)
 583      */
 584     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 585                                             final VrfEntry vrfEntry) {
 586         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 587         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 588         // packet is commuted from Vpn2 to Vpn1.
 589         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 590         if (!interVpnLink.isActive()) {
 591             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 592                      interVpnLinkName, vrfEntry.getDestPrefix());
 593             return;
 594         }
 595
 596         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 597         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 598         if (!optLportTag.isPresent()) {
 599             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 600             return;
 601         }
 602
 603         Long lportTag = optLportTag.get();
 604         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 605         if (label == null) {
 606             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 607                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 608             return;
 609         }
 610         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 611         List<InstructionInfo> instructions = Arrays.asList(
 612             new InstructionApplyActions(actionsInfos),
 613             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 614                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 615                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 616                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 617             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 618         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 619
 620         for (BigInteger dpId : targetDpns) {
 621             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 622                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 623
 624             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 625                                /*writeTx*/null);
 626         }
 627     }
 628
 629
 630     /*
 631      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 632      */
 633     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 634                                                final VrfEntry vrfEntry, long vpnTag) {
 635         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 636         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 637             && vrfEntry.getRoutePaths().size() == 1);
 638         String destination = vrfEntry.getDestPrefix();
 639         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 640         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 641
 642         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 643         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 644         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 645         if (interVpnLink.getState().or(State.Error) != State.Active) {
 646             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 647                 destination, nextHop, interVpnLinkName);
 648             return;
 649         }
 650
 651         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 652         if (!optOtherEndpointLportTag.isPresent()) {
 653             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 654                 vpnUuid, interVpnLinkName);
 655             return;
 656         }
 657
 658         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 659         if (targetDpns.isEmpty()) {
 660             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 661                 vpnUuid, interVpnLinkName);
 662             return;
 663         }
 664
 665         String[] values = destination.split("/");
 666         String destPrefixIpAddress = values[0];
 667         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 668
 669         List<MatchInfo> matches = new ArrayList<>();
 670         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 671         matches.add(MatchEthernetType.IPV4);
 672
 673         if (prefixLength != 0) {
 674             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 675         }
 676
 677         List<Instruction> instructions =
 678             Arrays.asList(new InstructionWriteMetadata(
 679                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 680                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 681                             .L3VPN_SERVICE_INDEX)),
 682                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 683                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 684
 685         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 686         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 687         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 688             COOKIE_VM_FIB_TABLE, matches, instructions);
 689
 690         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 691             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 692
 693         for (BigInteger dpId : targetDpns) {
 694
 695             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 696                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 697                 dpId, interVpnLink.getInterVpnLinkName());
 698
 699             mdsalManager.installFlow(dpId, flowEntity);
 700         }
 701     }
 702
 703     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 704         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 705         String localNextHopIP = vrfEntry.getDestPrefix();
 706         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 707         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 708         if (localNextHopInfo == null) {
 709             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 710             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 711                     vpnName, usedRds, localNextHopIP);
 712             boolean localNextHopSeen = false;
 713             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 714             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 715                 String ipPrefix;
 716                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 717                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 718                 } else {
 719                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 720                 }
 721                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 722                     ipPrefix);
 723                 if (localNextHopInfoLocal != null) {
 724                     localNextHopSeen = true;
 725                     BigInteger dpnId =
 726                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 727                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes);
 728                     returnLocalDpnId.add(dpnId);
 729                 }
 730             }
 731             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 732                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 733                 if (optionalLabel.isPresent()) {
 734                     Long label = optionalLabel.get();
 735                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 736                     synchronized (label.toString().intern()) {
 737                         LabelRouteInfo lri = getLabelRouteInfo(label);
 738                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 739                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 740                                     fibUtil.getVpnInstanceOpData(rd);
 741                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 742                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 743                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 744                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 745                                     localNextHopIP = lri.getPrefix();
 746                                 } else {
 747                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 748                                     localNextHopIP = lri.getPrefix();
 749                                 }
 750                             }
 751                             if (localNextHopInfo != null) {
 752                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 753                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 754                                 if (vpnExtraRoutes.isEmpty()) {
 755                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 756                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes);
 757                                     returnLocalDpnId.add(dpnId);
 758                                 } else {
 759                                     for (Routes extraRoutes : vpnExtraRoutes) {
 760                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 761                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes);
 762                                         returnLocalDpnId.add(dpnId);
 763                                     }
 764                                 }
 765                             }
 766                         }
 767                     }
 768                 }
 769             }
 770             if (returnLocalDpnId.isEmpty()) {
 771                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 772             }
 773         } else {
 774             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 775                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null);
 776             if (dpnId != null) {
 777                 returnLocalDpnId.add(dpnId);
 778             }
 779         }
 780         return returnLocalDpnId;
 781     }
 782
 783     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 784                                                 final Long vpnId, final String rd,
 785                                                 final VrfEntry vrfEntry,
 786                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 787         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 788         if (localNextHopInfo != null) {
 789             long groupId;
 790             long localGroupId;
 791             final BigInteger dpnId = localNextHopInfo.getDpnId();
 792             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 793                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 794                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 795                 return dpnId;
 796             }
 797             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 798                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 799                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 800                 return dpnId;
 801             }
 802             if (!isVpnPresentInDpn(rd, dpnId)) {
 803                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 804                         vpnId, rd, dpnId.toString());
 805                 return BigInteger.ZERO;
 806             }
 807             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 808             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 809             String prefix = vrfEntry.getDestPrefix();
 810             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 811             //The loadbalancing group is created only if the extra route has multiple nexthops
 812             //to avoid loadbalancing the discovered routes
 813             if (vpnExtraRoutes != null && routes != null) {
 814                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 815                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 816                 } else {
 817                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 818                 }
 819                 if (vpnExtraRoutes.size() > 1) {
 820                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 821                             vpnExtraRoutes);
 822                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 823                 } else if (routes.getNexthopIpList().size() > 1) {
 824                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 825                             vpnExtraRoutes);
 826                     localGroupId = groupId;
 827                 } else {
 828                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 829                             prefix, gwMacAddress, jobKey);
 830                     localGroupId = groupId;
 831                 }
 832             } else {
 833                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 834                         gwMacAddress, jobKey);
 835                 localGroupId = groupId;
 836             }
 837             if (groupId == FibConstants.INVALID_GROUP_ID) {
 838                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 839                         prefix, rd, interfaceName, dpnId.toString());
 840                 return BigInteger.ZERO;
 841             }
 842             final List<InstructionInfo> instructions = Collections.singletonList(
 843                     new InstructionApplyActions(
 844                             Collections.singletonList(new ActionGroup(groupId))));
 845             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 846                     new InstructionApplyActions(
 847                             Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 848             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 849             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 850             jobCoordinator.enqueueJob(jobKey, () -> {
 851                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 852                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 853                             NwConstants.ADD_FLOW, tx, null);
 854                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 855                             vpnName, rd)) {
 856                         optLabel.ifPresent(label -> {
 857                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 858                                 LOG.debug(
 859                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 860                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 861                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 862                                         nextHopAddressList);
 863                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 864                                         NwConstants.ADD_FLOW, tx);
 865                                 // If the extra-route is reachable from VMs attached to the same switch,
 866                                 // then the tunnel table can point to the load balancing group.
 867                                 // If it is reachable from VMs attached to different switches,
 868                                 // then it should be pointing to one of the local group in order to avoid looping.
 869                                 if (vrfEntry.getRoutePaths().size() == 1) {
 870                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 871                                 } else {
 872                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 873                                 }
 874                             } else {
 875                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 876                                                 + "route. LFib and Terminating table entries will not be created.",
 877                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 878                             }
 879                         });
 880                     }
 881                 }));
 882             });
 883             return dpnId;
 884         }
 885         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 886                 vpnName);
 887         return BigInteger.ZERO;
 888     }
 889
 890     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 891         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 892         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 893         if (dpnInVpn.isPresent()) {
 894             return true;
 895         }
 896         return false;
 897     }
 898
 899     private LabelRouteInfo getLabelRouteInfo(Long label) {
 900         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 901             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 902         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 903         if (opResult.isPresent()) {
 904             return opResult.get();
 905         }
 906         return null;
 907     }
 908
 909     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 910         if (lri == null) {
 911             return true;
 912         }
 913
 914         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 915         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 916             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 917
 918         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 919             ? lri.getVpnInstanceList() : new ArrayList<>();
 920         if (vpnInstancesList.contains(vpnInstanceName)) {
 921             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 922             vpnInstancesList.remove(vpnInstanceName);
 923         }
 924         if (vpnInstancesList.isEmpty()) {
 925             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 926             if (tx != null) {
 927                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 928             } else {
 929                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 930             }
 931             return true;
 932         } else {
 933             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 934             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 935             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 936         }
 937         return false;
 938     }
 939
 940     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 941                                       WriteTransaction tx) {
 942         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 943
 944         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 945
 946         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 947             dpId, label, groupId);
 948     }
 949
 950     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 951                                                 WriteTransaction tx) {
 952         List<MatchInfo> mkMatches = new ArrayList<>();
 953
 954         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 955             destDpId, label, actionsInfos);
 956
 957         // Matching metadata
 958         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 959         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 960
 961         List<InstructionInfo> mkInstructions = new ArrayList<>();
 962         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 963
 964         FlowEntity terminatingServiceTableFlowEntity =
 965             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 966             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 967                 String.format("%s:%d", "TST Flow Entry ", label),
 968             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 969
 970         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 971
 972         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 973
 974         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 975         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 976             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 977             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 978             .child(Flow.class, flowKey).build();
 979         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
 980                 WriteTransaction.CREATE_MISSING_PARENTS);
 981     }
 982
 983     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 984         FlowEntity flowEntity;
 985         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
 986         List<MatchInfo> mkMatches = new ArrayList<>();
 987         // Matching metadata
 988         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 989         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 990             NwConstants.INTERNAL_TUNNEL_TABLE,
 991             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
 992             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
 993             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 994         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
 995         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 996         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 997             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 998             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 999
1000         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1001         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1002     }
1003
1004     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1005         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1006         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1007         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1008         boolean shouldUpdateNonEcmpLocalNextHop = true;
1009         if (localNextHopInfo == null) {
1010             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1011             if (usedRds.size() > 1) {
1012                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1013                         vrfEntry.getDestPrefix(), vpnName, rd);
1014                 return returnLocalDpnId;
1015             }
1016             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1017             //in the vpn
1018             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1019                     vpnName, rd, vrfEntry.getDestPrefix());
1020             if (extraRouteOptional.isPresent()) {
1021                 Routes extraRoute = extraRouteOptional.get();
1022                 String ipPrefix;
1023                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1024                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1025                 } else {
1026                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1027                 }
1028                 if (extraRoute.getNexthopIpList().size() > 1) {
1029                     shouldUpdateNonEcmpLocalNextHop = false;
1030                 }
1031                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1032                 if (localNextHopInfo != null) {
1033                     String localNextHopIP = localNextHopInfo.getIpAddress();
1034                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1035                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1036                     if (!dpnId.equals(BigInteger.ZERO)) {
1037                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1038                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1039                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1040                                 /*remove*/ false);
1041                         returnLocalDpnId.add(dpnId);
1042                     }
1043                 } else {
1044                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1045                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1046                 }
1047             }
1048
1049             if (localNextHopInfo == null) {
1050                 /* Imported VRF entry */
1051                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1052                 if (optionalLabel.isPresent()) {
1053                     Long label = optionalLabel.get();
1054                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1055                     LabelRouteInfo lri = getLabelRouteInfo(label);
1056                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1057                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1058                         prefixBuilder.setDpnId(lri.getDpnId());
1059                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1060                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1061                         if (!dpnId.equals(BigInteger.ZERO)) {
1062                             returnLocalDpnId.add(dpnId);
1063                         }
1064                     }
1065                 }
1066             }
1067
1068         } else {
1069             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1070             String localNextHopIP = localNextHopInfo.getIpAddress();
1071             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1072                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1073             if (!dpnId.equals(BigInteger.ZERO)) {
1074                 returnLocalDpnId.add(dpnId);
1075             }
1076         }
1077
1078         return returnLocalDpnId;
1079     }
1080
1081     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1082                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1083                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1084         if (localNextHopInfo != null) {
1085             final BigInteger dpnId = localNextHopInfo.getDpnId();
1086             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1087                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1088                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1089                 return dpnId;
1090             }
1091             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1092                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1093                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1094                 return dpnId;
1095             }
1096
1097             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1098                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1099                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1100                             NwConstants.DEL_FLOW, tx, null);
1101                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1102                             vpnId, rd)) {
1103                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1104                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1105                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1106                                         NwConstants.DEL_FLOW, tx);
1107                                 removeTunnelTableEntry(dpnId, label, tx);
1108                             });
1109                         }
1110                     }
1111                 })));
1112             //TODO: verify below adjacency call need to be optimized (?)
1113             //In case of the removal of the extra route, the loadbalancing group is updated
1114             if (shouldUpdateNonEcmpLocalNextHop) {
1115                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1116             }
1117             return dpnId;
1118         }
1119         return BigInteger.ZERO;
1120     }
1121
1122     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1123             final VrfEntry vrfEntry, WriteTransaction tx) {
1124         if (tx == null) {
1125             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1126                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1127             }), LOG, "Error creating remote FIB entry");
1128             return;
1129         }
1130
1131         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1132         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1133                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1134
1135         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1136         if (adjacencyResults.isEmpty()) {
1137             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1138                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1139             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1140             return;
1141         }
1142
1143         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1144         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1145                 vpnName, usedRds, vrfEntry.getDestPrefix());
1146         // create loadbalancing groups for extra routes only when the extra route is present behind
1147         // multiple VMs
1148         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1149                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1150             List<InstructionInfo> instructions = new ArrayList<>();
1151             // Obtain the local routes for this particular dpn.
1152             java.util.Optional<Routes> routes = vpnExtraRoutes
1153                     .stream()
1154                     .filter(route -> {
1155                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1156                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1157                         if (prefixToInterface == null) {
1158                             return false;
1159                         }
1160                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1161                     }).findFirst();
1162             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1163                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1164             if (groupId == FibConstants.INVALID_GROUP_ID) {
1165                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1166                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1167                 return;
1168             }
1169             List<ActionInfo> actionInfos =
1170                     Collections.singletonList(new ActionGroup(groupId));
1171             instructions.add(new InstructionApplyActions(actionInfos));
1172             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1173                     NwConstants.ADD_FLOW, tx, null);
1174         } else {
1175             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1176         }
1177
1178         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1179     }
1180
1181     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1182     /* Get interface info from prefix to interface mapping;
1183         Use the interface info to get the corresponding vpn interface op DS entry,
1184         remove the adjacency corresponding to this fib entry.
1185         If adjacency removed is the last adjacency, clean up the following:
1186          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1187          - prefix to interface entry
1188          - vpn interface op DS
1189      */
1190         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1191         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1192         if (prefixInfo == null) {
1193             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1194             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1195             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1196             if (extraRoute != null) {
1197                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1198                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1199                     if (nextHopIp != null) {
1200                         String ipPrefix;
1201                         if (isIpv4Address(nextHopIp)) {
1202                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1203                         } else {
1204                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1205                         }
1206                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1207                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1208                     }
1209                 }
1210             }
1211             if (prefixInfo == null) {
1212                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1213                 if (optionalLabel.isPresent()) {
1214                     Long label = optionalLabel.get();
1215                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1216                     LabelRouteInfo lri = getLabelRouteInfo(label);
1217                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1218                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1219                         prefixBuilder.setDpnId(lri.getDpnId());
1220                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1221                         prefixBuilder.setIpAddress(lri.getPrefix());
1222                         prefixInfo = prefixBuilder.build();
1223                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1224                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1225                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1226                     }
1227                 }
1228             }
1229         } else {
1230             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1231         }
1232     }
1233
1234     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1235                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1236
1237         if (prefixInfo == null) {
1238             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1239                     vrfEntry.getDestPrefix(), vpnId, rd);
1240             return; //Don't have any info for this prefix (shouldn't happen); need to return
1241         }
1242
1243         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1244             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1245                     vrfEntry.getDestPrefix(), vpnId, rd);
1246             return;
1247         }
1248
1249         String ifName = prefixInfo.getVpnInterfaceName();
1250         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1251             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1252     }
1253
1254     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1255         Prefixes prefixInfo;
1256         Long vpnId;
1257         String rd;
1258         VrfEntry vrfEntry;
1259         Routes extraRoute;
1260
1261         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1262                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1263             this.prefixInfo = prefixInfo;
1264             this.vpnId = vpnId;
1265             this.rd = rd;
1266             this.vrfEntry = vrfEntry;
1267             this.extraRoute = extraRoute;
1268         }
1269
1270         @Override
1271         public List<ListenableFuture<Void>> call() {
1272             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1273             // to call the respective helpers.
1274             return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1275                 //First Cleanup LabelRouteInfo
1276                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1277                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1278                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1279                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1280                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1281                         synchronized (label.toString().intern()) {
1282                             LabelRouteInfo lri = getLabelRouteInfo(label);
1283                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1284                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1285                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1286                                         fibUtil.getVpnInstanceOpData(rd);
1287                                 String vpnInstanceName = "";
1288                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1289                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1290                                 }
1291                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1292                                 if (lriRemoved) {
1293                                     String parentRd = lri.getParentVpnRd();
1294                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1295                                             parentRd, vrfEntry.getDestPrefix()));
1296                                 }
1297                             } else {
1298                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1299                                         rd, vrfEntry.getDestPrefix()));
1300                             }
1301                         }
1302                     });
1303                 }
1304                 String ifName = prefixInfo.getVpnInterfaceName();
1305                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1306                 String vpnName = null;
1307
1308                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1309                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1310                     * */
1311                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1312                     if (vpnNameOpt.isPresent()) {
1313                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1314                     }
1315                 }
1316                 if (optVpnName.isPresent()) {
1317                     vpnName = optVpnName.get();
1318                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1319                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1320                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1321                     if (opVpnInterface.isPresent()) {
1322                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1323                         if (vpnId != associatedVpnId) {
1324                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1325                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1326                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1327                             return;
1328                         } else {
1329                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1330                                     vrfEntry.getDestPrefix(), associatedVpnId);
1331                         }
1332                     }
1333                 }
1334                 if (extraRoute != null) {
1335                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1336                     //Only one used Rd present in case of removal event
1337                     String usedRd = usedRds.get(0);
1338                     if (optVpnName.isPresent()) {
1339                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1340                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1341                                         vrfEntry.getDestPrefix()));
1342                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1343                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1344                     }
1345                 }
1346                 Optional<AdjacenciesOp> optAdjacencies =
1347                     MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1348                                    FibUtil.getAdjListPathOp(ifName, vpnName));
1349                 int numAdj = 0;
1350                 if (optAdjacencies.isPresent()) {
1351                     numAdj = optAdjacencies.get().getAdjacency().size();
1352                 }
1353                 //remove adjacency corr to prefix
1354                 if (numAdj > 1) {
1355                     LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {} vpnName {}", vpnId,
1356                              vrfEntry.getDestPrefix(), vpnName);
1357                     tx.delete(LogicalDatastoreType.OPERATIONAL,
1358                               FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1359                 } else {
1360                     //this is last adjacency (or) no more adjacency left for this vpn interface, so
1361                     //clean up the vpn interface from DpnToVpn list
1362                     LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.",
1363                              ifName, prefixInfo.getDpnId(), rd);
1364                     tx.delete(LogicalDatastoreType.OPERATIONAL,
1365                               FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1366                 }
1367             }));
1368         }
1369     }
1370
1371     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1372         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1373         final String rd = vrfTableKey.getRouteDistinguisher();
1374         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1375         if (vpnInstance == null) {
1376             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1377             return;
1378         }
1379         final Collection<VpnToDpnList> vpnToDpnList;
1380         if (vrfEntry.getParentVpnRd() != null
1381                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1382             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1383             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1384             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1385                     vpnInstance.getVpnToDpnList();
1386             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1387                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1388         } else {
1389             vpnToDpnList = vpnInstance.getVpnToDpnList();
1390         }
1391
1392         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1393         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1394         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1395         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1396         if (subnetRoute != null) {
1397             long elanTag = subnetRoute.getElantag();
1398             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1399                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1400             if (vpnToDpnList != null) {
1401                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1402                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1403                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1404
1405                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1406                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1407                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1408                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1409                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1410                             }
1411
1412                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1413                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1414                         }
1415                     })));
1416             }
1417             optionalLabel.ifPresent(label -> {
1418                 synchronized (label.toString().intern()) {
1419                     LabelRouteInfo lri = getLabelRouteInfo(label);
1420                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1421                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1422                                 fibUtil.getVpnInstanceOpData(rd);
1423                         String vpnInstanceName = "";
1424                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1425                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1426                         }
1427                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1428                         if (lriRemoved) {
1429                             String parentRd = lri.getParentVpnRd();
1430                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1431                                     parentRd, vrfEntry.getDestPrefix()));
1432                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1433                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1434                         }
1435                     } else {
1436                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1437                                 rd, vrfEntry.getDestPrefix()));
1438                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1439                                 label, rd, vrfEntry.getDestPrefix());
1440                     }
1441                 }
1442             });
1443             return;
1444         }
1445
1446         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1447             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1448         if (vpnToDpnList != null) {
1449             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1450                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1451             String jobKey;
1452             Optional<Routes> extraRouteOptional;
1453             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1454             if (usedRds != null && !usedRds.isEmpty()) {
1455                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1456                 if (usedRds.size() > 1) {
1457                     LOG.error("The extra route prefix is still present in some DPNs");
1458                     return ;
1459                 } else {
1460                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1461                     //is not present in any other DPN
1462                     extraRouteOptional = VpnExtraRouteHelper
1463                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1464                 }
1465             } else {
1466                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1467                 extraRouteOptional = Optional.absent();
1468             }
1469
1470             jobCoordinator.enqueueJob(jobKey,
1471                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1472                     if (localDpnIdList.size() <= 0) {
1473                         for (VpnToDpnList curDpn : vpnToDpnList) {
1474                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1475                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1476                         }
1477                     } else {
1478                         for (BigInteger localDpnId : localDpnIdList) {
1479                             for (VpnToDpnList curDpn : vpnToDpnList) {
1480                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1481                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1482                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1483                                 }
1484                             }
1485                         }
1486                     }
1487                 })), MAX_RETRIES);
1488         }
1489
1490         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1491         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1492         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1493
1494         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1495         // of the interVpnLink.
1496         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1497         if (optVpnUuid.isPresent()) {
1498             String vpnUuid = optVpnUuid.get();
1499             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1500                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1501                 if (optInterVpnLink.isPresent()) {
1502                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1503                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1504                         // This is route that points to the other endpoint of an InterVpnLink
1505                         // In that case, we should look for the FIB table pointing to
1506                         // LPortDispatcher table and remove it.
1507                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1508                     }
1509                 }
1510             });
1511         }
1512
1513     }
1514
1515     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1516                                     int addOrRemove, WriteTransaction tx) {
1517         if (tx == null) {
1518             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1519                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1520                 "Error making LFIB table entry");
1521             return;
1522         }
1523
1524         List<MatchInfo> matches = new ArrayList<>();
1525         matches.add(MatchEthernetType.MPLS_UNICAST);
1526         matches.add(new MatchMplsLabel(label));
1527
1528         // Install the flow entry in L3_LFIB_TABLE
1529         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1530
1531         FlowEntity flowEntity;
1532         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1533             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1534         Flow flow = flowEntity.getFlowBuilder().build();
1535         String flowId = flowEntity.getFlowId();
1536         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1537         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1538         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1539             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1540             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1541
1542         if (addOrRemove == NwConstants.ADD_FLOW) {
1543             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1544         } else {
1545             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1546         }
1547
1548         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1549             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1550     }
1551
1552     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1553                                     final FutureCallback<List<Void>> callback) {
1554         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1555         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1556             () -> {
1557                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1558                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1559                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1560                         LogicalDatastoreType.CONFIGURATION, id);
1561                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1562                 if (!vrfTable.isPresent()) {
1563                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1564                     if (callback != null) {
1565                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1566                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1567                     }
1568                     return futures;
1569                 }
1570                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1571                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1572                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1573                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1574                             if (subnetRoute != null) {
1575                                 long elanTag = subnetRoute.getElantag();
1576                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1577                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1578                                         tx);
1579                                 continue;
1580                             }
1581                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1582                             if (routerInt != null) {
1583                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1584                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1585                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1586                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1587                                         NwConstants.ADD_FLOW);
1588                                 continue;
1589                             }
1590                             //Handle local flow creation for imports
1591                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1592                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1593                                 if (optionalLabel.isPresent()) {
1594                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1595                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1596                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1597                                         if (lri.getDpnId().equals(dpnId)) {
1598                                             createLocalFibEntry(vpnId, rd, vrfEntry);
1599                                             continue;
1600                                         }
1601                                     }
1602                                 }
1603                             }
1604                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1605                                     vrfEntry, dpnId);
1606                             if (shouldCreateRemoteFibEntry) {
1607                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1608                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1609                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1610                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1611                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1612                                 } else {
1613                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1614                                             vrfEntry, tx);
1615                                 }
1616                             }
1617                         }
1618                     }));
1619                     if (callback != null) {
1620                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1621                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1622                     }
1623                 }
1624                 return futures;
1625             });
1626     }
1627
1628     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1629                                             final String localNextHopIp, final String remoteNextHopIp) {
1630         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1631             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1632         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1633         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1634         List<SubTransaction> txnObjects =  new ArrayList<>();
1635         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1636         if (vrfTable.isPresent()) {
1637             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1638                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1639                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1640                         vrfTable.get().getVrfEntry().stream()
1641                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1642                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1643                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1644                     }
1645                 })));
1646         }
1647     }
1648
1649     public void manageRemoteRouteOnDPN(final boolean action,
1650                                        final BigInteger localDpnId,
1651                                        final long vpnId,
1652                                        final String rd,
1653                                        final String destPrefix,
1654                                        final String destTepIp,
1655                                        final long label) {
1656         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1657
1658         if (vpnInstance == null) {
1659             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1660             return;
1661         }
1662
1663         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1664             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1665                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1666                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1667                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1668                     if (vrfEntry == null) {
1669                         return;
1670                     }
1671                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1672                             action, localDpnId, vpnId, rd, destPrefix);
1673                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1674                     VrfEntry modVrfEntry;
1675                     if (routePathList == null || routePathList.isEmpty()) {
1676                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1677                                 Collections.singletonList(destTepIp),
1678                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1679                     } else {
1680                         modVrfEntry = vrfEntry;
1681                     }
1682
1683                     if (action) {
1684                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1685                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1686                                 modVrfEntry, tx);
1687                     } else {
1688                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1689                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1690                                 vrfEntry.getDestPrefix());
1691                         if (usedRds.size() > 1) {
1692                             LOG.debug("The extra route prefix is still present in some DPNs");
1693                             return;
1694                         }
1695                         //Is this fib route an extra route? If yes, get the nexthop which would be
1696                         //an adjacency in the vpn
1697                         Optional<Routes> extraRouteOptional = Optional.absent();
1698                         if (usedRds.size() != 0) {
1699                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1700                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1701                                     usedRds.get(0), vrfEntry.getDestPrefix());
1702                         }
1703                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1704                                 extraRouteOptional, tx);
1705                     }
1706                 }
1707             })));
1708     }
1709
1710     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1711                                  final FutureCallback<List<Void>> callback) {
1712         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1713         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1714             () -> {
1715                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1716                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1717                 List<SubTransaction> txnObjects =  new ArrayList<>();
1718                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1719                         LogicalDatastoreType.CONFIGURATION, id);
1720                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1721                 if (vrfTable.isPresent()) {
1722                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1723                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1724                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1725                                 /* Handle subnet routes here */
1726                                 SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1727                                 if (subnetRoute != null) {
1728                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1729                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1730                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1731                                             NwConstants.DEL_FLOW, tx, null);
1732                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1733                                     if (routePaths != null) {
1734                                         for (RoutePaths routePath : routePaths) {
1735                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1736                                                     DEFAULT_FIB_FLOW_PRIORITY,
1737                                                     NwConstants.DEL_FLOW, tx);
1738                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1739                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1740                                                     vrfEntry.getDestPrefix());
1741                                         }
1742                                     }
1743                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1744                                             NwConstants.DEL_FLOW, tx);
1745                                     continue;
1746                                 }
1747                                 // ping responder for router interfaces
1748                                 RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1749                                 if (routerInt != null) {
1750                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1751                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1752                                             routerInt.getMacAddress());
1753                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1754                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1755                                             NwConstants.DEL_FLOW);
1756                                     continue;
1757                                 }
1758
1759                                 //Handle local flow deletion for imports
1760                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1761                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1762                                     if (optionalLabel.isPresent()) {
1763                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1764                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1765                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1766                                                 lri) && lri.getDpnId().equals(dpnId)) {
1767                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1768                                         }
1769                                     }
1770                                 }
1771
1772                                 // Passing null as we don't know the dpn
1773                                 // to which prefix is attached at this point
1774                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1775                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1776                                 String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1777                                 Optional<Routes> extraRouteOptional;
1778                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1779                                 //an adjacency in the vpn
1780                                 if (usedRds != null && !usedRds.isEmpty()) {
1781                                     if (usedRds.size() > 1) {
1782                                         LOG.error("The extra route prefix is still present in some DPNs");
1783                                         return;
1784                                     } else {
1785                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1786                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1787
1788                                     }
1789                                 } else {
1790                                     extraRouteOptional = Optional.absent();
1791                                 }
1792                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1793                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1794                                             vrfTable.get().getKey(), vrfEntry, extraRouteOptional, tx, txnObjects);
1795                                 } else {
1796                                     baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
1797                                             vrfEntry, extraRouteOptional, tx);
1798                                 }
1799                             }
1800                         }));
1801                     }
1802                     if (callback != null) {
1803                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1804                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1805                     }
1806                 } else {
1807                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1808                 }
1809                 return futures;
1810             });
1811
1812     }
1813
1814     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1815                                            final String localNextHopIp, final String remoteNextHopIp) {
1816         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1817                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1818             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1819         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1820         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1821         List<SubTransaction> txnObjects =  new ArrayList<>();
1822         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1823         if (vrfTable.isPresent()) {
1824             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1825                 () -> {
1826                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1827                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1828                             tx -> vrfTable.get().getVrfEntry().stream()
1829                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1830                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1831                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1832                     }
1833                 });
1834         }
1835     }
1836
1837     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1838         InstanceIdentifierBuilder<VrfTables> idBuilder =
1839             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1840         return idBuilder.build();
1841     }
1842
1843     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1844         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1845                 .FLOWID_SEPARATOR + nextHop;
1846     }
1847
1848     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1849         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1850                 + tableMiss + FLOWID_PREFIX;
1851     }
1852
1853     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1854         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1855             .child(VrfTables.class, new VrfTablesKey(rd))
1856             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1857         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1858         if (vrfEntry.isPresent()) {
1859             return vrfEntry.get();
1860         }
1861         return null;
1862     }
1863
1864     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1865                                              final String vpnName,
1866                                              final VrfEntry vrfEntry) {
1867         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1868
1869         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1870         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1871
1872         if (targetDpns.isEmpty()) {
1873             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1874             return;
1875         }
1876
1877         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1878         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1879
1880         // delete from FIB
1881         //
1882         optNextHop.ifPresent(nextHop -> {
1883             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1884             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1885             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
1886                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1887
1888             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1889             for (BigInteger dpId : targetDpns) {
1890                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1891                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1892
1893                 mdsalManager.removeFlow(dpId, flow);
1894             }
1895         });
1896
1897         // delete from LFIB
1898         //
1899         optLabel.ifPresent(label -> {
1900             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1901
1902             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1903                 for (BigInteger dpId : targetDpns) {
1904                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1905                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1906                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1907                             tx);
1908                 }
1909             }), LOG, "Error removing flows");
1910         });
1911     }
1912
1913     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1914             List<String> nextHopAddressList, LabelRouteInfo lri) {
1915         return lri != null && lri.getPrefix().equals(prefix)
1916                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1917     }
1918
1919     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1920         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1921             return true;
1922         }
1923
1924         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1925         if (prefix != null) {
1926             BigInteger prefixDpnId = prefix.getDpnId();
1927             if (dpnId.equals(prefixDpnId)) {
1928                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1929                         vrfEntry, dpnId);
1930                 return false;
1931             }
1932         }
1933         return true;
1934     }
1935 }