fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  34 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  35 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  37 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  38 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  39 import org.opendaylight.genius.mdsalutil.ActionInfo;
  40 import org.opendaylight.genius.mdsalutil.FlowEntity;
  41 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.MatchInfo;
  44 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  49 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  50 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  51 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  52 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  53 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  54 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  55 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  56 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  58 import org.opendaylight.genius.utils.ServiceIndex;
  59 import org.opendaylight.genius.utils.batching.SubTransaction;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  62 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  63 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  64 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  65 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  66 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  67 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  68 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  69 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 101 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 102 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 103 import org.slf4j.Logger;
 104 import org.slf4j.LoggerFactory;
 105
 106
 107 @Singleton
 108 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 109
 110     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 111     private static final String FLOWID_PREFIX = "L3.";
 112     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 113     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 114     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 115     private static final int LFIB_INTERVPN_PRIORITY = 15;
 116     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 117     private static final int MAX_RETRIES = 3;
 118     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 123     private final IMdsalApiManager mdsalManager;
 124     private final NexthopManager nextHopManager;
 125     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 126     private final BaseVrfEntryHandler baseVrfEntryHandler;
 127     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 128     private final JobCoordinator jobCoordinator;
 129     private final IElanService elanManager;
 130     private final FibUtil fibUtil;
 131     private final InterVpnLinkCache interVpnLinkCache;
 132     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 133
 134     @Inject
 135     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 136                             final NexthopManager nexthopManager,
 137                             final IElanService elanManager,
 138                             final BaseVrfEntryHandler vrfEntryHandler,
 139                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 140                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 141                             final JobCoordinator jobCoordinator,
 142                             final FibUtil fibUtil,
 143                             final InterVpnLinkCache interVpnLinkCache) {
 144         super(VrfEntry.class, VrfEntryListener.class);
 145         this.dataBroker = dataBroker;
 146         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 147         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 148         this.mdsalManager = mdsalApiManager;
 149         this.nextHopManager = nexthopManager;
 150         this.elanManager = elanManager;
 151         this.baseVrfEntryHandler = vrfEntryHandler;
 152         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 153         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 154         this.jobCoordinator = jobCoordinator;
 155         this.fibUtil = fibUtil;
 156         this.interVpnLinkCache = interVpnLinkCache;
 157     }
 158
 159     @Override
 160     @PostConstruct
 161     public void init() {
 162         LOG.info("{} init", getClass().getSimpleName());
 163         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 164     }
 165
 166     @Override
 167     @SuppressWarnings("checkstyle:IllegalCatch")
 168     public void close() {
 169         closeables.forEach(c -> {
 170             try {
 171                 c.close();
 172             } catch (Exception e) {
 173                 LOG.warn("Error closing {}", c, e);
 174             }
 175         });
 176     }
 177
 178     @Override
 179     protected VrfEntryListener getDataTreeChangeListener() {
 180         return VrfEntryListener.this;
 181     }
 182
 183     @Override
 184     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 185         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 186     }
 187
 188     @Override
 189     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 190         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 191         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 192         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 193                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 194         addFibEntries(identifier, vrfEntry, rd);
 195         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 196                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 197     }
 198
 199     //This method is temporary. Eventually Factory design pattern will be used to get
 200     // right VrfEntryhandle and invoke its methods.
 201     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 202         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 203             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 204             return;
 205         }
 206         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 207             LOG.info("EVPN flows need to be programmed.");
 208             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 209                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 210             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 211             closeables.add(evpnVrfEntryHandler);
 212             return;
 213         }
 214         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 215         if (routerInt != null) {
 216             // ping responder for router interfaces
 217             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 218             return;
 219         }
 220         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 221             createFibEntries(identifier, vrfEntry);
 222             return;
 223         }
 224     }
 225
 226     @Override
 227     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 228         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 229         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 230         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 231                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 232         removeFibEntries(identifier, vrfEntry, rd);
 233         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 234             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 235     }
 236
 237     //This method is temporary. Eventually Factory design pattern will be used to get
 238     // right VrfEntryhandle and invoke its methods.
 239     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 240         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 241             LOG.info("EVPN flows to be deleted");
 242             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 243                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 244             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 245             closeables.add(evpnVrfEntryHandler);
 246             return;
 247         }
 248         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 249         if (routerInt != null) {
 250             // ping responder for router interfaces
 251             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 252             return;
 253         }
 254         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 255             deleteFibEntries(identifier, vrfEntry);
 256             return;
 257         }
 258         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 259             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 260             return;
 261         }
 262     }
 263
 264     @Override
 265     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 266     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 267     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 268     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 269         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 270         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 271         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 272                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 273         // Handle BGP Routes first
 274         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 275             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 276             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 277                     rd, update.getDestPrefix(), update.getRoutePaths());
 278             return;
 279         }
 280
 281         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 282             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 283             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 284             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 285
 286             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 287             //updated vrf entry route path is null. This can happen during tunnel events.
 288             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 289             List<String> usedRds = new ArrayList<>();
 290             if (optVpnInstance.isPresent()) {
 291                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 292                         update.getDestPrefix());
 293             }
 294             // If original VRF Entry had nexthop null , but update VRF Entry
 295             // has nexthop , route needs to be created on remote Dpns
 296             if (originalRoutePath == null || originalRoutePath.isEmpty()
 297                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 298                 // TODO(vivek): Though ugly, Not handling this code now, as each
 299                 // tep add event will invoke flow addition
 300                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 301                         + " This event is IGNORED here.", update.getDestPrefix());
 302                 return;
 303             }
 304
 305             // If original VRF Entry had valid nexthop , but update VRF Entry
 306             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 307             if (updateRoutePath == null || updateRoutePath.isEmpty()
 308                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 309                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 310                         + "This event is IGNORED here.", update.getDestPrefix());
 311                 return;
 312             }
 313             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 314             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 315             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 316             ListenableFuture<Void> future =
 317                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> nextHopsRemoved.parallelStream()
 318                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 319                                     tx, nextHopRemoved, rd, update.getDestPrefix())));
 320             Futures.addCallback(future, new FutureCallback<Void>() {
 321                 @Override
 322                 public void onSuccess(Void result) {
 323                     createFibEntries(identifier, update);
 324                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 325                             rd, update.getDestPrefix(), update.getRoutePaths());
 326                 }
 327
 328                 @Override
 329                 public void onFailure(Throwable throwable) {
 330                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 331                             update, throwable);
 332                 }
 333             }, MoreExecutors.directExecutor());
 334         }
 335
 336         //Handle all other routes only on a cluster reboot
 337         if (original.equals(update)) {
 338             //Reboot use-case
 339             createFibEntries(identifier, update);
 340             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 341                     rd, update.getDestPrefix(), update.getRoutePaths());
 342             return;
 343         }
 344
 345         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 346                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 347     }
 348
 349     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 350         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 351         List<SubTransaction> txnObjects =  new ArrayList<>();
 352         final VpnInstanceOpDataEntry vpnInstance =
 353                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 354         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 355         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 356                 + " has null vpnId!");
 357         final Collection<VpnToDpnList> vpnToDpnList;
 358         if (vrfEntry.getParentVpnRd() != null
 359                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 360             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 361             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 362             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 363                 vpnInstance.getVpnToDpnList();
 364             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 365                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 366         } else {
 367             vpnToDpnList = vpnInstance.getVpnToDpnList();
 368         }
 369         final Long vpnId = vpnInstance.getVpnId();
 370         final String rd = vrfTableKey.getRouteDistinguisher();
 371         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 372         if (subnetRoute != null) {
 373             final long elanTag = subnetRoute.getElantag();
 374             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 375                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 376             if (vpnToDpnList != null) {
 377                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 378                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 379                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 380                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 381                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 382                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 383                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 384                             }
 385                         }
 386                     })));
 387             }
 388             return;
 389         }
 390
 391         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 392         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 393             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 394                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 395                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 396                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 397                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 398                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 399                                     try {
 400                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 401                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 402                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 403                                                     txnObjects);
 404                                         } else {
 405                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 406                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 407                                         }
 408                                     } catch (NullPointerException e) {
 409                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 410                                                 vrfEntry.getDestPrefix(), e);
 411                                     }
 412                                 }
 413                             }
 414                         }
 415                     }
 416                 })), MAX_RETRIES);
 417         }
 418
 419         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 420         if (optVpnUuid.isPresent()) {
 421             String vpnUuid = optVpnUuid.get();
 422             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 423             if (interVpnLink != null) {
 424                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 425                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 426                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 427                         // This is an static route that points to the other endpoint of an InterVpnLink
 428                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 429                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 430                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
 431                     }
 432                 });
 433             }
 434         }
 435     }
 436
 437     void refreshFibTables(String rd, String prefix) {
 438         InstanceIdentifier<VrfEntry> vrfEntryId =
 439                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 440                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 441         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 442         if (vrfEntry.isPresent()) {
 443             createFibEntries(vrfEntryId, vrfEntry.get());
 444         }
 445     }
 446
 447     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 448         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 449         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 450         prefixBuilder.setDpnId(lri.getDpnId());
 451         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 452         prefixBuilder.setIpAddress(lri.getPrefix());
 453         // Increment the refCount here
 454         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 455             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 456         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 457         if (!isPresentInList) {
 458             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 459             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 460             vpnInstanceNames.add(vpnInstanceName);
 461             builder.setVpnInstanceList(vpnInstanceNames);
 462             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 463         } else {
 464             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 465         }
 466         return prefixBuilder.build();
 467     }
 468
 469     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 470                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 471         if (tx == null) {
 472             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 473                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 474                 "Error installing subnet route in FIB");
 475             return;
 476         }
 477         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 478             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 479             synchronized (label.toString().intern()) {
 480                 LabelRouteInfo lri = getLabelRouteInfo(label);
 481                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 482
 483                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 484                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 485                                 fibUtil.getVpnInstanceOpData(rd);
 486                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 487                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 488                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 489                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 490                             }
 491                         }
 492                     }
 493                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 494                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 495                 }
 496             }
 497         });
 498         final List<InstructionInfo> instructions = new ArrayList<>();
 499         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 500             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 501         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 502         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 503         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 504                 NwConstants.ADD_FLOW, tx, null);
 505
 506         if (vrfEntry.getRoutePaths() != null) {
 507             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 508                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 509                     List<ActionInfo> actionsInfos = new ArrayList<>();
 510                     // reinitialize instructions list for LFIB Table
 511                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 512
 513                     actionsInfos.add(new ActionPopMpls());
 514                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 515                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 516                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 517                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 518
 519                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 520                             NwConstants.ADD_FLOW, tx);
 521                 }
 522             }
 523         }
 524     }
 525
 526     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 527                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 528         List<MatchInfo> matches = new ArrayList<>();
 529
 530         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 531                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 532         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 533         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 534         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 535
 536         InetAddress destPrefix;
 537         try {
 538             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 539         } catch (UnknownHostException e) {
 540             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 541                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 542             return;
 543         }
 544
 545         // Match on VpnId and SubnetBroadCast IP address
 546         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 547         matches.add(MatchEthernetType.IPV4);
 548
 549         if (prefixLength != 0) {
 550             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 551         }
 552
 553         //Action is to drop the packet
 554         List<InstructionInfo> dropInstructions = new ArrayList<>();
 555         List<ActionInfo> actionsInfos = new ArrayList<>();
 556         actionsInfos.add(new ActionDrop());
 557         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 558
 559         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 560         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 561         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 562                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 563
 564         Flow flow = flowEntity.getFlowBuilder().build();
 565         String flowId = flowEntity.getFlowId();
 566         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 567         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 568
 569         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 570                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 571                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 572
 573         if (addOrRemove == NwConstants.ADD_FLOW) {
 574             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 575         } else {
 576             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 577         }
 578     }
 579
 580     /*
 581      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 582      * LportDispatcher table (via table 80)
 583      */
 584     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 585                                             final VrfEntry vrfEntry) {
 586         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 587         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 588         // packet is commuted from Vpn2 to Vpn1.
 589         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 590         if (!interVpnLink.isActive()) {
 591             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 592                      interVpnLinkName, vrfEntry.getDestPrefix());
 593             return;
 594         }
 595
 596         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 597         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 598         if (!optLportTag.isPresent()) {
 599             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 600             return;
 601         }
 602
 603         Long lportTag = optLportTag.get();
 604         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 605         if (label == null) {
 606             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 607                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 608             return;
 609         }
 610         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 611         List<InstructionInfo> instructions = Arrays.asList(
 612             new InstructionApplyActions(actionsInfos),
 613             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 614                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 615                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 616                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 617             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 618         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 619
 620         for (BigInteger dpId : targetDpns) {
 621             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 622                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 623
 624             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 625                                /*writeTx*/null);
 626         }
 627     }
 628
 629
 630     /*
 631      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 632      */
 633     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 634                                                final VrfEntry vrfEntry, long vpnTag) {
 635         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 636         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 637             && vrfEntry.getRoutePaths().size() == 1);
 638         String destination = vrfEntry.getDestPrefix();
 639         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 640         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 641
 642         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 643         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 644         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 645         if (interVpnLink.getState().or(State.Error) != State.Active) {
 646             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 647                 destination, nextHop, interVpnLinkName);
 648             return;
 649         }
 650
 651         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 652         if (!optOtherEndpointLportTag.isPresent()) {
 653             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 654                 vpnUuid, interVpnLinkName);
 655             return;
 656         }
 657
 658         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 659         if (targetDpns.isEmpty()) {
 660             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 661                 vpnUuid, interVpnLinkName);
 662             return;
 663         }
 664
 665         String[] values = destination.split("/");
 666         String destPrefixIpAddress = values[0];
 667         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 668
 669         List<MatchInfo> matches = new ArrayList<>();
 670         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 671         matches.add(MatchEthernetType.IPV4);
 672
 673         if (prefixLength != 0) {
 674             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 675         }
 676
 677         List<Instruction> instructions =
 678             Arrays.asList(new InstructionWriteMetadata(
 679                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 680                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 681                             .L3VPN_SERVICE_INDEX)),
 682                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 683                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 684
 685         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 686         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 687         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 688             COOKIE_VM_FIB_TABLE, matches, instructions);
 689
 690         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 691             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 692
 693         for (BigInteger dpId : targetDpns) {
 694
 695             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 696                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 697                 dpId, interVpnLink.getInterVpnLinkName());
 698
 699             mdsalManager.installFlow(dpId, flowEntity);
 700         }
 701     }
 702
 703     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 704         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 705         String localNextHopIP = vrfEntry.getDestPrefix();
 706         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 707         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 708         if (localNextHopInfo == null) {
 709             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 710             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 711                     vpnName, usedRds, localNextHopIP);
 712             boolean localNextHopSeen = false;
 713             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 714             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 715                 String ipPrefix;
 716                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 717                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 718                 } else {
 719                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 720                 }
 721                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 722                     ipPrefix);
 723                 if (localNextHopInfoLocal != null) {
 724                     localNextHopSeen = true;
 725                     BigInteger dpnId =
 726                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 727                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes);
 728                     returnLocalDpnId.add(dpnId);
 729                 }
 730             }
 731             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 732                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 733                 if (optionalLabel.isPresent()) {
 734                     Long label = optionalLabel.get();
 735                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 736                     synchronized (label.toString().intern()) {
 737                         LabelRouteInfo lri = getLabelRouteInfo(label);
 738                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 739                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 740                                     fibUtil.getVpnInstanceOpData(rd);
 741                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 742                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 743                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 744                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 745                                     localNextHopIP = lri.getPrefix();
 746                                 } else {
 747                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 748                                     localNextHopIP = lri.getPrefix();
 749                                 }
 750                             }
 751                             if (localNextHopInfo != null) {
 752                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 753                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 754                                 if (vpnExtraRoutes.isEmpty()) {
 755                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 756                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes);
 757                                     returnLocalDpnId.add(dpnId);
 758                                 } else {
 759                                     for (Routes extraRoutes : vpnExtraRoutes) {
 760                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 761                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes);
 762                                         returnLocalDpnId.add(dpnId);
 763                                     }
 764                                 }
 765                             }
 766                         }
 767                     }
 768                 }
 769             }
 770             if (returnLocalDpnId.isEmpty()) {
 771                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 772             }
 773         } else {
 774             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 775                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null);
 776             if (dpnId != null) {
 777                 returnLocalDpnId.add(dpnId);
 778             }
 779         }
 780         return returnLocalDpnId;
 781     }
 782
 783     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 784                                                 final Long vpnId, final String rd,
 785                                                 final VrfEntry vrfEntry,
 786                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 787         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 788         if (localNextHopInfo != null) {
 789             long groupId;
 790             long localGroupId;
 791             final BigInteger dpnId = localNextHopInfo.getDpnId();
 792             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 793                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 794                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 795                 return dpnId;
 796             }
 797             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 798                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 799                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 800                 return dpnId;
 801             }
 802             if (!isVpnPresentInDpn(rd, dpnId)) {
 803                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 804                         vpnId, rd, dpnId.toString());
 805                 return BigInteger.ZERO;
 806             }
 807             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 808             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 809             String prefix = vrfEntry.getDestPrefix();
 810             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 811             //The loadbalancing group is created only if the extra route has multiple nexthops
 812             //to avoid loadbalancing the discovered routes
 813             if (vpnExtraRoutes != null && routes != null) {
 814                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 815                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 816                 } else {
 817                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 818                 }
 819                 if (vpnExtraRoutes.size() > 1) {
 820                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 821                             vpnExtraRoutes);
 822                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 823                 } else if (routes.getNexthopIpList().size() > 1) {
 824                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 825                             vpnExtraRoutes);
 826                     localGroupId = groupId;
 827                 } else {
 828                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 829                             prefix, gwMacAddress, jobKey);
 830                     localGroupId = groupId;
 831                 }
 832             } else {
 833                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 834                         gwMacAddress, jobKey);
 835                 localGroupId = groupId;
 836             }
 837             if (groupId == FibConstants.INVALID_GROUP_ID) {
 838                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 839                         prefix, rd, interfaceName, dpnId.toString());
 840                 return BigInteger.ZERO;
 841             }
 842             final List<InstructionInfo> instructions = Collections.singletonList(
 843                     new InstructionApplyActions(
 844                             Collections.singletonList(new ActionGroup(groupId))));
 845             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 846                     new InstructionApplyActions(
 847                             Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 848             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 849             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 850             jobCoordinator.enqueueJob(jobKey, () -> {
 851                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 852                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 853                             NwConstants.ADD_FLOW, tx, null);
 854                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 855                             vpnName, rd)) {
 856                         optLabel.ifPresent(label -> {
 857                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 858                                 LOG.debug(
 859                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 860                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 861                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 862                                         nextHopAddressList);
 863                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 864                                         NwConstants.ADD_FLOW, tx);
 865                                 // If the extra-route is reachable from VMs attached to the same switch,
 866                                 // then the tunnel table can point to the load balancing group.
 867                                 // If it is reachable from VMs attached to different switches,
 868                                 // then it should be pointing to one of the local group in order to avoid looping.
 869                                 if (vrfEntry.getRoutePaths().size() == 1) {
 870                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 871                                 } else {
 872                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 873                                 }
 874                             } else {
 875                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 876                                                 + "route. LFib and Terminating table entries will not be created.",
 877                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 878                             }
 879                         });
 880                     }
 881                 }));
 882             });
 883             return dpnId;
 884         }
 885         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 886                 vpnName);
 887         return BigInteger.ZERO;
 888     }
 889
 890     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 891         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 892         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 893         if (dpnInVpn.isPresent()) {
 894             return true;
 895         }
 896         return false;
 897     }
 898
 899     private LabelRouteInfo getLabelRouteInfo(Long label) {
 900         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 901             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 902         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 903         if (opResult.isPresent()) {
 904             return opResult.get();
 905         }
 906         return null;
 907     }
 908
 909     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 910         if (lri == null) {
 911             return true;
 912         }
 913
 914         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 915         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 916             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 917
 918         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 919             ? lri.getVpnInstanceList() : new ArrayList<>();
 920         if (vpnInstancesList.contains(vpnInstanceName)) {
 921             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 922             vpnInstancesList.remove(vpnInstanceName);
 923         }
 924         if (vpnInstancesList.isEmpty()) {
 925             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 926             if (tx != null) {
 927                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 928             } else {
 929                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 930             }
 931             return true;
 932         } else {
 933             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 934             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 935             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 936         }
 937         return false;
 938     }
 939
 940     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 941                                       WriteTransaction tx) {
 942         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 943
 944         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 945
 946         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 947             dpId, label, groupId);
 948     }
 949
 950     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 951                                                 WriteTransaction tx) {
 952         List<MatchInfo> mkMatches = new ArrayList<>();
 953
 954         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 955             destDpId, label, actionsInfos);
 956
 957         // Matching metadata
 958         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 959         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 960
 961         List<InstructionInfo> mkInstructions = new ArrayList<>();
 962         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 963
 964         FlowEntity terminatingServiceTableFlowEntity =
 965             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 966             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 967                 String.format("%s:%d", "TST Flow Entry ", label),
 968             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 969
 970         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 971
 972         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 973
 974         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 975         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 976             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 977             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 978             .child(Flow.class, flowKey).build();
 979         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
 980                 WriteTransaction.CREATE_MISSING_PARENTS);
 981     }
 982
 983     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 984         FlowEntity flowEntity;
 985         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
 986         List<MatchInfo> mkMatches = new ArrayList<>();
 987         // Matching metadata
 988         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 989         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 990             NwConstants.INTERNAL_TUNNEL_TABLE,
 991             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
 992             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
 993             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 994         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
 995         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 996         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 997             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 998             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 999
1000         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1001         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1002     }
1003
1004     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1005         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1006         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1007         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1008         boolean shouldUpdateNonEcmpLocalNextHop = true;
1009         if (localNextHopInfo == null) {
1010             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1011             if (usedRds.size() > 1) {
1012                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1013                         vrfEntry.getDestPrefix(), vpnName, rd);
1014                 return returnLocalDpnId;
1015             }
1016             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1017             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1018             //in the vpn
1019             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1020                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1021             if (extraRouteOptional.isPresent()) {
1022                 Routes extraRoute = extraRouteOptional.get();
1023                 String ipPrefix;
1024                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1025                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1026                 } else {
1027                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1028                 }
1029                 if (extraRoute.getNexthopIpList().size() > 1) {
1030                     shouldUpdateNonEcmpLocalNextHop = false;
1031                 }
1032                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1033                 if (localNextHopInfo != null) {
1034                     String localNextHopIP = localNextHopInfo.getIpAddress();
1035                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1036                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1037                     if (!dpnId.equals(BigInteger.ZERO)) {
1038                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1039                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1040                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1041                                 /*remove*/ false);
1042                         returnLocalDpnId.add(dpnId);
1043                     }
1044                 } else {
1045                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1046                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1047                 }
1048             }
1049
1050             if (localNextHopInfo == null) {
1051                 /* Imported VRF entry */
1052                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1053                 if (optionalLabel.isPresent()) {
1054                     Long label = optionalLabel.get();
1055                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1056                     LabelRouteInfo lri = getLabelRouteInfo(label);
1057                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1058                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1059                         prefixBuilder.setDpnId(lri.getDpnId());
1060                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1061                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1062                         if (!dpnId.equals(BigInteger.ZERO)) {
1063                             returnLocalDpnId.add(dpnId);
1064                         }
1065                     }
1066                 }
1067             }
1068
1069         } else {
1070             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1071             String localNextHopIP = localNextHopInfo.getIpAddress();
1072             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1073                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1074             if (!dpnId.equals(BigInteger.ZERO)) {
1075                 returnLocalDpnId.add(dpnId);
1076             }
1077         }
1078
1079         return returnLocalDpnId;
1080     }
1081
1082     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1083                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1084                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1085         if (localNextHopInfo != null) {
1086             final BigInteger dpnId = localNextHopInfo.getDpnId();
1087             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1088                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1089                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1090                 return dpnId;
1091             }
1092             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1093                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1094                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1095                 return dpnId;
1096             }
1097
1098             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1099                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1100                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1101                             NwConstants.DEL_FLOW, tx, null);
1102                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1103                             vpnId, rd)) {
1104                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1105                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1106                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1107                                         NwConstants.DEL_FLOW, tx);
1108                                 removeTunnelTableEntry(dpnId, label, tx);
1109                             });
1110                         }
1111                     }
1112                 })));
1113             //TODO: verify below adjacency call need to be optimized (?)
1114             //In case of the removal of the extra route, the loadbalancing group is updated
1115             if (shouldUpdateNonEcmpLocalNextHop) {
1116                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1117             }
1118             return dpnId;
1119         }
1120         return BigInteger.ZERO;
1121     }
1122
1123     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1124             final VrfEntry vrfEntry, WriteTransaction tx) {
1125         if (tx == null) {
1126             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1127                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1128             }), LOG, "Error creating remote FIB entry");
1129             return;
1130         }
1131
1132         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1133         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1134                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1135
1136         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1137         if (adjacencyResults.isEmpty()) {
1138             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1139                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1140             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1141             return;
1142         }
1143
1144         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1145         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1146                 vpnName, usedRds, vrfEntry.getDestPrefix());
1147         // create loadbalancing groups for extra routes only when the extra route is present behind
1148         // multiple VMs
1149         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1150                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1151             List<InstructionInfo> instructions = new ArrayList<>();
1152             // Obtain the local routes for this particular dpn.
1153             java.util.Optional<Routes> routes = vpnExtraRoutes
1154                     .stream()
1155                     .filter(route -> {
1156                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1157                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1158                         if (prefixToInterface == null) {
1159                             return false;
1160                         }
1161                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1162                     }).findFirst();
1163             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1164                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1165             if (groupId == FibConstants.INVALID_GROUP_ID) {
1166                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1167                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1168                 return;
1169             }
1170             List<ActionInfo> actionInfos =
1171                     Collections.singletonList(new ActionGroup(groupId));
1172             instructions.add(new InstructionApplyActions(actionInfos));
1173             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1174                     NwConstants.ADD_FLOW, tx, null);
1175         } else {
1176             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1177         }
1178
1179         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1180     }
1181
1182     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1183     /* Get interface info from prefix to interface mapping;
1184         Use the interface info to get the corresponding vpn interface op DS entry,
1185         remove the adjacency corresponding to this fib entry.
1186         If adjacency removed is the last adjacency, clean up the following:
1187          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1188          - prefix to interface entry
1189          - vpn interface op DS
1190      */
1191         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1192         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1193         if (prefixInfo == null) {
1194             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1195             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1196             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1197             if (extraRoute != null) {
1198                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1199                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1200                     if (nextHopIp != null) {
1201                         String ipPrefix;
1202                         if (isIpv4Address(nextHopIp)) {
1203                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1204                         } else {
1205                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1206                         }
1207                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1208                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1209                     }
1210                 }
1211             }
1212             if (prefixInfo == null) {
1213                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1214                 if (optionalLabel.isPresent()) {
1215                     Long label = optionalLabel.get();
1216                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1217                     LabelRouteInfo lri = getLabelRouteInfo(label);
1218                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1219                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1220                         prefixBuilder.setDpnId(lri.getDpnId());
1221                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1222                         prefixBuilder.setIpAddress(lri.getPrefix());
1223                         prefixInfo = prefixBuilder.build();
1224                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1225                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1226                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1227                     }
1228                 }
1229             }
1230         } else {
1231             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1232         }
1233     }
1234
1235     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1236                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1237
1238         if (prefixInfo == null) {
1239             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1240                     vrfEntry.getDestPrefix(), vpnId, rd);
1241             return; //Don't have any info for this prefix (shouldn't happen); need to return
1242         }
1243
1244         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1245             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1246                     vrfEntry.getDestPrefix(), vpnId, rd);
1247             return;
1248         }
1249
1250         String ifName = prefixInfo.getVpnInterfaceName();
1251         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1252             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1253     }
1254
1255     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1256         Prefixes prefixInfo;
1257         Long vpnId;
1258         String rd;
1259         VrfEntry vrfEntry;
1260         Routes extraRoute;
1261
1262         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1263                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1264             this.prefixInfo = prefixInfo;
1265             this.vpnId = vpnId;
1266             this.rd = rd;
1267             this.vrfEntry = vrfEntry;
1268             this.extraRoute = extraRoute;
1269         }
1270
1271         @Override
1272         public List<ListenableFuture<Void>> call() {
1273             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1274             // to call the respective helpers.
1275             return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1276                 //First Cleanup LabelRouteInfo
1277                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1278                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1279                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1280                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1281                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1282                         synchronized (label.toString().intern()) {
1283                             LabelRouteInfo lri = getLabelRouteInfo(label);
1284                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1285                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1286                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1287                                         fibUtil.getVpnInstanceOpData(rd);
1288                                 String vpnInstanceName = "";
1289                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1290                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1291                                 }
1292                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1293                                 if (lriRemoved) {
1294                                     String parentRd = lri.getParentVpnRd();
1295                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1296                                             parentRd, vrfEntry.getDestPrefix()));
1297                                 }
1298                             } else {
1299                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1300                                         rd, vrfEntry.getDestPrefix()));
1301                             }
1302                         }
1303                     });
1304                 }
1305                 String ifName = prefixInfo.getVpnInterfaceName();
1306                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1307                 String vpnName = null;
1308
1309                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1310                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1311                     * */
1312                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1313                     if (vpnNameOpt.isPresent()) {
1314                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1315                     }
1316                 }
1317                 if (optVpnName.isPresent()) {
1318                     vpnName = optVpnName.get();
1319                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1320                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1321                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1322                     if (opVpnInterface.isPresent()) {
1323                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1324                         if (vpnId != associatedVpnId) {
1325                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1326                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1327                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1328                             return;
1329                         } else {
1330                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1331                                     vrfEntry.getDestPrefix(), associatedVpnId);
1332                         }
1333                     }
1334                 }
1335                 if (extraRoute != null) {
1336                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1337                     //Only one used Rd present in case of removal event
1338                     String usedRd = usedRds.get(0);
1339                     if (optVpnName.isPresent()) {
1340                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1341                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1342                                         vrfEntry.getDestPrefix()));
1343                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1344                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1345                     }
1346                 }
1347                 Optional<AdjacenciesOp> optAdjacencies =
1348                     MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1349                                    FibUtil.getAdjListPathOp(ifName, vpnName));
1350                 int numAdj = 0;
1351                 if (optAdjacencies.isPresent()) {
1352                     numAdj = optAdjacencies.get().getAdjacency().size();
1353                 }
1354                 //remove adjacency corr to prefix
1355                 if (numAdj > 1) {
1356                     LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {} vpnName {}", vpnId,
1357                              vrfEntry.getDestPrefix(), vpnName);
1358                     tx.delete(LogicalDatastoreType.OPERATIONAL,
1359                               FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1360                 } else {
1361                     //this is last adjacency (or) no more adjacency left for this vpn interface, so
1362                     //clean up the vpn interface from DpnToVpn list
1363                     LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.",
1364                              ifName, prefixInfo.getDpnId(), rd);
1365                     tx.delete(LogicalDatastoreType.OPERATIONAL,
1366                               FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1367                 }
1368             }));
1369         }
1370     }
1371
1372     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1373         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1374         final String rd = vrfTableKey.getRouteDistinguisher();
1375         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1376         if (vpnInstance == null) {
1377             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1378             return;
1379         }
1380         final Collection<VpnToDpnList> vpnToDpnList;
1381         if (vrfEntry.getParentVpnRd() != null
1382                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1383             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1384             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1385             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1386                     vpnInstance.getVpnToDpnList();
1387             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1388                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1389         } else {
1390             vpnToDpnList = vpnInstance.getVpnToDpnList();
1391         }
1392
1393         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1394         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1395         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1396         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1397         if (subnetRoute != null) {
1398             long elanTag = subnetRoute.getElantag();
1399             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1400                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1401             if (vpnToDpnList != null) {
1402                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1403                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1404                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1405
1406                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1407                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1408                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1409                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1410                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1411                             }
1412
1413                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1414                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1415                         }
1416                     })));
1417             }
1418             optionalLabel.ifPresent(label -> {
1419                 synchronized (label.toString().intern()) {
1420                     LabelRouteInfo lri = getLabelRouteInfo(label);
1421                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1422                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1423                                 fibUtil.getVpnInstanceOpData(rd);
1424                         String vpnInstanceName = "";
1425                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1426                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1427                         }
1428                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1429                         if (lriRemoved) {
1430                             String parentRd = lri.getParentVpnRd();
1431                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1432                                     parentRd, vrfEntry.getDestPrefix()));
1433                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1434                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1435                         }
1436                     } else {
1437                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1438                                 rd, vrfEntry.getDestPrefix()));
1439                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1440                                 label, rd, vrfEntry.getDestPrefix());
1441                     }
1442                 }
1443             });
1444             return;
1445         }
1446
1447         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1448             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1449         if (vpnToDpnList != null) {
1450             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1451                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1452             String jobKey;
1453             Optional<Routes> extraRouteOptional;
1454             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1455             if (usedRds != null && !usedRds.isEmpty()) {
1456                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1457                 if (usedRds.size() > 1) {
1458                     LOG.error("The extra route prefix is still present in some DPNs");
1459                     return ;
1460                 } else {
1461                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1462                     //is not present in any other DPN
1463                     extraRouteOptional = VpnExtraRouteHelper
1464                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1465                 }
1466             } else {
1467                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1468                 extraRouteOptional = Optional.absent();
1469             }
1470
1471             jobCoordinator.enqueueJob(jobKey,
1472                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1473                     if (localDpnIdList.size() <= 0) {
1474                         for (VpnToDpnList curDpn : vpnToDpnList) {
1475                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1476                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1477                         }
1478                     } else {
1479                         for (BigInteger localDpnId : localDpnIdList) {
1480                             for (VpnToDpnList curDpn : vpnToDpnList) {
1481                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1482                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1483                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1484                                 }
1485                             }
1486                         }
1487                     }
1488                 })), MAX_RETRIES);
1489         }
1490
1491         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1492         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1493         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1494
1495         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1496         // of the interVpnLink.
1497         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1498         if (optVpnUuid.isPresent()) {
1499             String vpnUuid = optVpnUuid.get();
1500             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1501                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1502                 if (optInterVpnLink.isPresent()) {
1503                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1504                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1505                         // This is route that points to the other endpoint of an InterVpnLink
1506                         // In that case, we should look for the FIB table pointing to
1507                         // LPortDispatcher table and remove it.
1508                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1509                     }
1510                 }
1511             });
1512         }
1513
1514     }
1515
1516     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1517                                     int addOrRemove, WriteTransaction tx) {
1518         if (tx == null) {
1519             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1520                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1521                 "Error making LFIB table entry");
1522             return;
1523         }
1524
1525         List<MatchInfo> matches = new ArrayList<>();
1526         matches.add(MatchEthernetType.MPLS_UNICAST);
1527         matches.add(new MatchMplsLabel(label));
1528
1529         // Install the flow entry in L3_LFIB_TABLE
1530         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1531
1532         FlowEntity flowEntity;
1533         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1534             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1535         Flow flow = flowEntity.getFlowBuilder().build();
1536         String flowId = flowEntity.getFlowId();
1537         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1538         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1539         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1540             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1541             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1542
1543         if (addOrRemove == NwConstants.ADD_FLOW) {
1544             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1545         } else {
1546             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1547         }
1548
1549         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1550             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1551     }
1552
1553     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1554                                     final FutureCallback<List<Void>> callback) {
1555         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1556         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1557             () -> {
1558                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1559                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1560                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1561                         LogicalDatastoreType.CONFIGURATION, id);
1562                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1563                 if (!vrfTable.isPresent()) {
1564                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1565                     if (callback != null) {
1566                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1567                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1568                     }
1569                     return futures;
1570                 }
1571                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1572                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1573                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1574                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1575                             if (subnetRoute != null) {
1576                                 long elanTag = subnetRoute.getElantag();
1577                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1578                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1579                                         tx);
1580                                 continue;
1581                             }
1582                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1583                             if (routerInt != null) {
1584                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1585                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1586                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1587                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1588                                         NwConstants.ADD_FLOW);
1589                                 continue;
1590                             }
1591                             //Handle local flow creation for imports
1592                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1593                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1594                                 if (optionalLabel.isPresent()) {
1595                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1596                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1597                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1598                                         if (lri.getDpnId().equals(dpnId)) {
1599                                             createLocalFibEntry(vpnId, rd, vrfEntry);
1600                                             continue;
1601                                         }
1602                                     }
1603                                 }
1604                             }
1605                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1606                                     vrfEntry, dpnId);
1607                             if (shouldCreateRemoteFibEntry) {
1608                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1609                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1610                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1611                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1612                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1613                                 } else {
1614                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1615                                             vrfEntry, tx);
1616                                 }
1617                             }
1618                         }
1619                     }));
1620                     if (callback != null) {
1621                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1622                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1623                     }
1624                 }
1625                 return futures;
1626             });
1627     }
1628
1629     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1630                                             final String localNextHopIp, final String remoteNextHopIp) {
1631         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1632             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1633         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1634         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1635         List<SubTransaction> txnObjects =  new ArrayList<>();
1636         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1637         if (vrfTable.isPresent()) {
1638             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1639                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1640                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1641                         vrfTable.get().getVrfEntry().stream()
1642                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1643                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1644                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1645                     }
1646                 })));
1647         }
1648     }
1649
1650     public void manageRemoteRouteOnDPN(final boolean action,
1651                                        final BigInteger localDpnId,
1652                                        final long vpnId,
1653                                        final String rd,
1654                                        final String destPrefix,
1655                                        final String destTepIp,
1656                                        final long label) {
1657         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1658
1659         if (vpnInstance == null) {
1660             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1661             return;
1662         }
1663
1664         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1665             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1666                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1667                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1668                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1669                     if (vrfEntry == null) {
1670                         return;
1671                     }
1672                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1673                             action, localDpnId, vpnId, rd, destPrefix);
1674                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1675                     VrfEntry modVrfEntry;
1676                     if (routePathList == null || routePathList.isEmpty()) {
1677                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1678                                 Collections.singletonList(destTepIp),
1679                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1680                     } else {
1681                         modVrfEntry = vrfEntry;
1682                     }
1683
1684                     if (action) {
1685                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1686                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1687                                 modVrfEntry, tx);
1688                     } else {
1689                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1690                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1691                                 vrfEntry.getDestPrefix());
1692                         if (usedRds.size() > 1) {
1693                             LOG.debug("The extra route prefix is still present in some DPNs");
1694                             return;
1695                         }
1696                         //Is this fib route an extra route? If yes, get the nexthop which would be
1697                         //an adjacency in the vpn
1698                         Optional<Routes> extraRouteOptional = Optional.absent();
1699                         if (usedRds.size() != 0) {
1700                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1701                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1702                                     usedRds.get(0), vrfEntry.getDestPrefix());
1703                         }
1704                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1705                                 extraRouteOptional, tx);
1706                     }
1707                 }
1708             })));
1709     }
1710
1711     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1712                                  final FutureCallback<List<Void>> callback) {
1713         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1714         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1715             () -> {
1716                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1717                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1718                 List<SubTransaction> txnObjects =  new ArrayList<>();
1719                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1720                         LogicalDatastoreType.CONFIGURATION, id);
1721                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1722                 if (vrfTable.isPresent()) {
1723                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1724                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1725                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1726                                 /* Handle subnet routes here */
1727                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1728                                 if (subnetRoute != null) {
1729                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1730                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1731                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1732                                             NwConstants.DEL_FLOW, tx, null);
1733                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1734                                     if (routePaths != null) {
1735                                         for (RoutePaths routePath : routePaths) {
1736                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1737                                                     DEFAULT_FIB_FLOW_PRIORITY,
1738                                                     NwConstants.DEL_FLOW, tx);
1739                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1740                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1741                                                     vrfEntry.getDestPrefix());
1742                                         }
1743                                     }
1744                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1745                                             NwConstants.DEL_FLOW, tx);
1746                                     continue;
1747                                 }
1748                                 // ping responder for router interfaces
1749                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1750                                 if (routerInt != null) {
1751                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1752                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1753                                             routerInt.getMacAddress());
1754                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1755                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1756                                             NwConstants.DEL_FLOW);
1757                                     continue;
1758                                 }
1759
1760                                 //Handle local flow deletion for imports
1761                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1762                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1763                                     if (optionalLabel.isPresent()) {
1764                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1765                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1766                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1767                                                 lri) && lri.getDpnId().equals(dpnId)) {
1768                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1769                                         }
1770                                     }
1771                                 }
1772
1773                                 // Passing null as we don't know the dpn
1774                                 // to which prefix is attached at this point
1775                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1776                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1777                                 String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1778                                 Optional<Routes> extraRouteOptional;
1779                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1780                                 //an adjacency in the vpn
1781                                 if (usedRds != null && !usedRds.isEmpty()) {
1782                                     if (usedRds.size() > 1) {
1783                                         LOG.error("The extra route prefix is still present in some DPNs");
1784                                         return;
1785                                     } else {
1786                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1787                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1788
1789                                     }
1790                                 } else {
1791                                     extraRouteOptional = Optional.absent();
1792                                 }
1793                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1794                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1795                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1796                                 } else {
1797                                     baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(),
1798                                             vrfEntry, extraRouteOptional, tx);
1799                                 }
1800                             }
1801                         }));
1802                     }
1803                     if (callback != null) {
1804                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1805                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1806                     }
1807                 } else {
1808                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1809                 }
1810                 return futures;
1811             });
1812
1813     }
1814
1815     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1816                                            final String localNextHopIp, final String remoteNextHopIp) {
1817         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1818                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1819             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1820         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1821         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1822         List<SubTransaction> txnObjects =  new ArrayList<>();
1823         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1824         if (vrfTable.isPresent()) {
1825             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1826                 () -> {
1827                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1828                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1829                             tx -> vrfTable.get().getVrfEntry().stream()
1830                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1831                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1832                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1833                     }
1834                 });
1835         }
1836     }
1837
1838     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1839         InstanceIdentifierBuilder<VrfTables> idBuilder =
1840             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1841         return idBuilder.build();
1842     }
1843
1844     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1845         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1846                 .FLOWID_SEPARATOR + nextHop;
1847     }
1848
1849     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1850         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1851                 + tableMiss + FLOWID_PREFIX;
1852     }
1853
1854     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1855         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1856             .child(VrfTables.class, new VrfTablesKey(rd))
1857             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1858         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1859         if (vrfEntry.isPresent()) {
1860             return vrfEntry.get();
1861         }
1862         return null;
1863     }
1864
1865     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1866                                              final String vpnName,
1867                                              final VrfEntry vrfEntry) {
1868         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1869
1870         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1871         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1872
1873         if (targetDpns.isEmpty()) {
1874             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1875             return;
1876         }
1877
1878         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1879         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1880
1881         // delete from FIB
1882         //
1883         optNextHop.ifPresent(nextHop -> {
1884             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1885             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1886             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1887                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1888
1889             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1890             for (BigInteger dpId : targetDpns) {
1891                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1892                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1893
1894                 mdsalManager.removeFlow(dpId, flow);
1895             }
1896         });
1897
1898         // delete from LFIB
1899         //
1900         optLabel.ifPresent(label -> {
1901             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1902
1903             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1904                 for (BigInteger dpId : targetDpns) {
1905                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1906                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1907                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1908                             tx);
1909                 }
1910             }), LOG, "Error removing flows");
1911         });
1912     }
1913
1914     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1915             List<String> nextHopAddressList, LabelRouteInfo lri) {
1916         return lri != null && lri.getPrefix().equals(prefix)
1917                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1918     }
1919
1920     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1921         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1922             return true;
1923         }
1924
1925         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1926         if (prefix != null) {
1927             BigInteger prefixDpnId = prefix.getDpnId();
1928             if (dpnId.equals(prefixDpnId)) {
1929                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1930                         vrfEntry, dpnId);
1931                 return false;
1932             }
1933         }
1934         return true;
1935     }
1936 }