fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  41 import org.opendaylight.genius.mdsalutil.ActionInfo;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  44 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  45 import org.opendaylight.genius.mdsalutil.MatchInfo;
  46 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  47 import org.opendaylight.genius.mdsalutil.NWUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  52 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  53 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  54 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  55 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  56 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  61 import org.opendaylight.genius.utils.ServiceIndex;
  62 import org.opendaylight.genius.utils.batching.SubTransaction;
  63 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  64 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  65 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  66 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  67 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  68 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  69 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  72 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111
 112 @Singleton
 113 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 114
 115     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 116     private static final String FLOWID_PREFIX = "L3.";
 117     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 118     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 119     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 120     private static final int LFIB_INTERVPN_PRIORITY = 15;
 121     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 122     private static final int MAX_RETRIES = 3;
 123     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 124
 125     private final DataBroker dataBroker;
 126     private final ManagedNewTransactionRunner txRunner;
 127     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 128     private final IMdsalApiManager mdsalManager;
 129     private final NexthopManager nextHopManager;
 130     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 131     private final BaseVrfEntryHandler baseVrfEntryHandler;
 132     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 133     private final JobCoordinator jobCoordinator;
 134     private final IElanService elanManager;
 135     private final FibUtil fibUtil;
 136     private final InterVpnLinkCache interVpnLinkCache;
 137     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 138
 139     @Inject
 140     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 141                             final NexthopManager nexthopManager,
 142                             final IElanService elanManager,
 143                             final BaseVrfEntryHandler vrfEntryHandler,
 144                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 145                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 146                             final JobCoordinator jobCoordinator,
 147                             final FibUtil fibUtil,
 148                             final InterVpnLinkCache interVpnLinkCache) {
 149         super(VrfEntry.class, VrfEntryListener.class);
 150         this.dataBroker = dataBroker;
 151         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 152         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 153         this.mdsalManager = mdsalApiManager;
 154         this.nextHopManager = nexthopManager;
 155         this.elanManager = elanManager;
 156         this.baseVrfEntryHandler = vrfEntryHandler;
 157         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 158         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 159         this.jobCoordinator = jobCoordinator;
 160         this.fibUtil = fibUtil;
 161         this.interVpnLinkCache = interVpnLinkCache;
 162     }
 163
 164     @Override
 165     @PostConstruct
 166     public void init() {
 167         LOG.info("{} init", getClass().getSimpleName());
 168         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 169     }
 170
 171     @Override
 172     @SuppressWarnings("checkstyle:IllegalCatch")
 173     public void close() {
 174         closeables.forEach(c -> {
 175             try {
 176                 c.close();
 177             } catch (Exception e) {
 178                 LOG.warn("Error closing {}", c, e);
 179             }
 180         });
 181     }
 182
 183     @Override
 184     protected VrfEntryListener getDataTreeChangeListener() {
 185         return VrfEntryListener.this;
 186     }
 187
 188     @Override
 189     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 190         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 191     }
 192
 193     @Override
 194     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 195         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 196         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 197         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 198                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 199         addFibEntries(identifier, vrfEntry, rd);
 200         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 201                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 202     }
 203
 204     //This method is temporary. Eventually Factory design pattern will be used to get
 205     // right VrfEntryhandle and invoke its methods.
 206     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 207         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 208             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 209             return;
 210         }
 211         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 212             LOG.info("EVPN flows need to be programmed.");
 213             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 214                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 215             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 216             closeables.add(evpnVrfEntryHandler);
 217             return;
 218         }
 219         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 220         if (routerInt != null) {
 221             // ping responder for router interfaces
 222             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 223             return;
 224         }
 225         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 226             createFibEntries(identifier, vrfEntry);
 227             return;
 228         }
 229     }
 230
 231     @Override
 232     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 233         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 234         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 235         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 236                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 237         removeFibEntries(identifier, vrfEntry, rd);
 238         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 239             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 240     }
 241
 242     //This method is temporary. Eventually Factory design pattern will be used to get
 243     // right VrfEntryhandle and invoke its methods.
 244     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 245         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 246             LOG.info("EVPN flows to be deleted");
 247             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 248                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 249             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 250             closeables.add(evpnVrfEntryHandler);
 251             return;
 252         }
 253         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 254         if (routerInt != null) {
 255             // ping responder for router interfaces
 256             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 257             return;
 258         }
 259         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 260             deleteFibEntries(identifier, vrfEntry);
 261             return;
 262         }
 263         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 264             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 265             return;
 266         }
 267     }
 268
 269     @Override
 270     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 271     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 272     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 273     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 274         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 275         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 276         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 277                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 278         // Handle BGP Routes first
 279         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 280             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 281             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 282                     rd, update.getDestPrefix(), update.getRoutePaths());
 283             return;
 284         }
 285
 286         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 287             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 288             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 289             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 290
 291             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 292             //updated vrf entry route path is null. This can happen during tunnel events.
 293             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 294             List<String> usedRds = new ArrayList<>();
 295             if (optVpnInstance.isPresent()) {
 296                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 297                         update.getDestPrefix());
 298             }
 299             // If original VRF Entry had nexthop null , but update VRF Entry
 300             // has nexthop , route needs to be created on remote Dpns
 301             if (originalRoutePath == null || originalRoutePath.isEmpty()
 302                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 303                 // TODO(vivek): Though ugly, Not handling this code now, as each
 304                 // tep add event will invoke flow addition
 305                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 306                         + " This event is IGNORED here.", update.getDestPrefix());
 307                 return;
 308             }
 309
 310             // If original VRF Entry had valid nexthop , but update VRF Entry
 311             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 312             if (updateRoutePath == null || updateRoutePath.isEmpty()
 313                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 314                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 315                         + "This event is IGNORED here.", update.getDestPrefix());
 316                 return;
 317             }
 318             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 319             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 320             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 321             ListenableFuture<Void> future =
 322                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> nextHopsRemoved.parallelStream()
 323                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 324                                     tx, nextHopRemoved, rd, update.getDestPrefix())));
 325             Futures.addCallback(future, new FutureCallback<Void>() {
 326                 @Override
 327                 public void onSuccess(Void result) {
 328                     createFibEntries(identifier, update);
 329                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 330                             rd, update.getDestPrefix(), update.getRoutePaths());
 331                 }
 332
 333                 @Override
 334                 public void onFailure(Throwable throwable) {
 335                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 336                             update, throwable);
 337                 }
 338             }, MoreExecutors.directExecutor());
 339             return;
 340         }
 341
 342         //Handle all other routes only on a cluster reboot
 343         if (original.equals(update)) {
 344             //Reboot use-case
 345             createFibEntries(identifier, update);
 346             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 347                     rd, update.getDestPrefix(), update.getRoutePaths());
 348             return;
 349         }
 350
 351         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 352                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 353     }
 354
 355     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 356         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 357         List<SubTransaction> txnObjects =  new ArrayList<>();
 358         final VpnInstanceOpDataEntry vpnInstance =
 359                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 360         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 361         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 362                 + " has null vpnId!");
 363         final Collection<VpnToDpnList> vpnToDpnList;
 364         if (vrfEntry.getParentVpnRd() != null
 365                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 366             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 367             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 368             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 369                 vpnInstance.getVpnToDpnList();
 370             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 371                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 372         } else {
 373             vpnToDpnList = vpnInstance.getVpnToDpnList();
 374         }
 375         final Long vpnId = vpnInstance.getVpnId();
 376         final String rd = vrfTableKey.getRouteDistinguisher();
 377         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 378         if (subnetRoute != null) {
 379             final long elanTag = subnetRoute.getElantag();
 380             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 381                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 382             if (vpnToDpnList != null) {
 383                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 384                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 385                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 386                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 387                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 388                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 389                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 390                             }
 391                         }
 392                     })));
 393             }
 394             return;
 395         }
 396         // Get etherType value based on the IpPrefix address family type
 397         int etherType;
 398         try {
 399             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 400         } catch (IllegalArgumentException ex) {
 401             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 402             return;
 403         }
 404
 405         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 406         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 407             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 408                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 409                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 410                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 411                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 412                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 413                                     try {
 414                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 415                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 416                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 417                                                     txnObjects);
 418                                         } else {
 419                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 420                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 421                                         }
 422                                     } catch (NullPointerException e) {
 423                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 424                                                 vrfEntry.getDestPrefix(), e);
 425                                     }
 426                                 }
 427                             }
 428                         }
 429                     }
 430                 })), MAX_RETRIES);
 431         }
 432
 433         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 434         if (optVpnUuid.isPresent()) {
 435             String vpnUuid = optVpnUuid.get();
 436             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 437             if (interVpnLink != null) {
 438                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 439                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 440                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 441                         // This is an static route that points to the other endpoint of an InterVpnLink
 442                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 443                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 444                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 445                     }
 446                 });
 447             }
 448         }
 449     }
 450
 451     void refreshFibTables(String rd, String prefix) {
 452         InstanceIdentifier<VrfEntry> vrfEntryId =
 453                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 454                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 455         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 456         if (vrfEntry.isPresent()) {
 457             createFibEntries(vrfEntryId, vrfEntry.get());
 458         }
 459     }
 460
 461     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 462         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 463         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 464         prefixBuilder.setDpnId(lri.getDpnId());
 465         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 466         prefixBuilder.setIpAddress(lri.getPrefix());
 467         // Increment the refCount here
 468         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 469             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 470         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 471         if (!isPresentInList) {
 472             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 473             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 474             vpnInstanceNames.add(vpnInstanceName);
 475             builder.setVpnInstanceList(vpnInstanceNames);
 476             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 477         } else {
 478             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 479         }
 480         return prefixBuilder.build();
 481     }
 482
 483     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 484                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 485         if (tx == null) {
 486             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 487                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 488                 "Error installing subnet route in FIB");
 489             return;
 490         }
 491         int etherType;
 492         try {
 493             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 494         } catch (IllegalArgumentException ex) {
 495             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 496             return;
 497         }
 498         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 499             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 500             synchronized (label.toString().intern()) {
 501                 LabelRouteInfo lri = getLabelRouteInfo(label);
 502                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 503
 504                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 505                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 506                                 fibUtil.getVpnInstanceOpData(rd);
 507                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 508                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 509                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 510                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 511                             }
 512                         }
 513                     }
 514                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 515                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 516                 }
 517             }
 518         });
 519         final List<InstructionInfo> instructions = new ArrayList<>();
 520         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 521             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 522         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 523         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 524         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 525                 NwConstants.ADD_FLOW, tx, null);
 526         if (vrfEntry.getRoutePaths() != null) {
 527             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 528                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 529                     List<ActionInfo> actionsInfos = new ArrayList<>();
 530                     // reinitialize instructions list for LFIB Table
 531                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 532                     actionsInfos.add(new ActionPopMpls(etherType));
 533                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 534                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 535                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 536                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 537
 538                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 539                             NwConstants.ADD_FLOW, tx);
 540                 }
 541             }
 542         }
 543     }
 544
 545     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 546                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 547         List<MatchInfo> matches = new ArrayList<>();
 548
 549         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 550                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 551         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 552         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 553         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 554
 555         InetAddress destPrefix;
 556         try {
 557             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 558         } catch (UnknownHostException e) {
 559             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 560                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 561             return;
 562         }
 563
 564         // Match on VpnId and SubnetBroadCast IP address
 565         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 566         matches.add(MatchEthernetType.IPV4);
 567
 568         if (prefixLength != 0) {
 569             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 570         }
 571
 572         //Action is to drop the packet
 573         List<InstructionInfo> dropInstructions = new ArrayList<>();
 574         List<ActionInfo> actionsInfos = new ArrayList<>();
 575         actionsInfos.add(new ActionDrop());
 576         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 577
 578         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 579         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 580         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 581                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 582
 583         Flow flow = flowEntity.getFlowBuilder().build();
 584         String flowId = flowEntity.getFlowId();
 585         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 586         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 587
 588         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 589                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 590                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 591
 592         if (addOrRemove == NwConstants.ADD_FLOW) {
 593             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 594         } else {
 595             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 596         }
 597     }
 598
 599     /*
 600      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 601      * LportDispatcher table (via table 80)
 602      */
 603     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 604                                             final VrfEntry vrfEntry, int etherType) {
 605         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 606         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 607         // packet is commuted from Vpn2 to Vpn1.
 608         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 609         if (!interVpnLink.isActive()) {
 610             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 611                      interVpnLinkName, vrfEntry.getDestPrefix());
 612             return;
 613         }
 614
 615         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 616         if (!optLportTag.isPresent()) {
 617             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 618             return;
 619         }
 620
 621         Long lportTag = optLportTag.get();
 622         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 623         if (label == null) {
 624             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 625                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 626             return;
 627         }
 628         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 629         List<InstructionInfo> instructions = Arrays.asList(
 630             new InstructionApplyActions(actionsInfos),
 631             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 632                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 633                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 634                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 635             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 636         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 637         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 638
 639         for (BigInteger dpId : targetDpns) {
 640             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 641                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 642
 643             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 644                                /*writeTx*/null);
 645         }
 646     }
 647
 648
 649     /*
 650      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 651      */
 652     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 653                                                final VrfEntry vrfEntry, long vpnTag) {
 654         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 655         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 656             && vrfEntry.getRoutePaths().size() == 1);
 657         String destination = vrfEntry.getDestPrefix();
 658         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 659         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 660
 661         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 662         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 663         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 664         if (interVpnLink.getState().or(State.Error) != State.Active) {
 665             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 666                 destination, nextHop, interVpnLinkName);
 667             return;
 668         }
 669
 670         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 671         if (!optOtherEndpointLportTag.isPresent()) {
 672             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 673                 vpnUuid, interVpnLinkName);
 674             return;
 675         }
 676
 677         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 678         if (targetDpns.isEmpty()) {
 679             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 680                 vpnUuid, interVpnLinkName);
 681             return;
 682         }
 683
 684         String[] values = destination.split("/");
 685         String destPrefixIpAddress = values[0];
 686         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 687
 688         List<MatchInfo> matches = new ArrayList<>();
 689         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 690         matches.add(MatchEthernetType.IPV4);
 691
 692         if (prefixLength != 0) {
 693             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 694         }
 695
 696         List<Instruction> instructions =
 697             Arrays.asList(new InstructionWriteMetadata(
 698                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 699                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 700                             .L3VPN_SERVICE_INDEX)),
 701                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 702                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 703
 704         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 705         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 706         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 707             COOKIE_VM_FIB_TABLE, matches, instructions);
 708
 709         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 710             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 711
 712         for (BigInteger dpId : targetDpns) {
 713
 714             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 715                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 716                 dpId, interVpnLink.getInterVpnLinkName());
 717
 718             mdsalManager.installFlow(dpId, flowEntity);
 719         }
 720     }
 721
 722     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 723         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 724         String localNextHopIP = vrfEntry.getDestPrefix();
 725         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 726         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 727         if (localNextHopInfo == null) {
 728             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 729             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 730                     vpnName, usedRds, localNextHopIP);
 731             boolean localNextHopSeen = false;
 732             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 733             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 734                 String ipPrefix;
 735                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 736                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 737                 } else {
 738                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 739                 }
 740                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 741                     ipPrefix);
 742                 if (localNextHopInfoLocal != null) {
 743                     localNextHopSeen = true;
 744                     BigInteger dpnId =
 745                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 746                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 747                     returnLocalDpnId.add(dpnId);
 748                 }
 749             }
 750             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 751                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 752                 if (optionalLabel.isPresent()) {
 753                     Long label = optionalLabel.get();
 754                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 755                     synchronized (label.toString().intern()) {
 756                         LabelRouteInfo lri = getLabelRouteInfo(label);
 757                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 758                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 759                                     fibUtil.getVpnInstanceOpData(rd);
 760                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 761                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 762                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 763                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 764                                     localNextHopIP = lri.getPrefix();
 765                                 } else {
 766                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 767                                     localNextHopIP = lri.getPrefix();
 768                                 }
 769                             }
 770                             if (localNextHopInfo != null) {
 771                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 772                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 773                                 if (vpnExtraRoutes.isEmpty()) {
 774                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 775                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 776                                     returnLocalDpnId.add(dpnId);
 777                                 } else {
 778                                     for (Routes extraRoutes : vpnExtraRoutes) {
 779                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 780                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 781                                         returnLocalDpnId.add(dpnId);
 782                                     }
 783                                 }
 784                             }
 785                         }
 786                     }
 787                 }
 788             }
 789             if (returnLocalDpnId.isEmpty()) {
 790                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 791             }
 792         } else {
 793             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 794                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 795             if (dpnId != null) {
 796                 returnLocalDpnId.add(dpnId);
 797             }
 798         }
 799         return returnLocalDpnId;
 800     }
 801
 802     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 803                                                 final Long vpnId, final String rd,
 804                                                 final VrfEntry vrfEntry,
 805                                                 Routes routes, List<Routes> vpnExtraRoutes,
 806                                                 int etherType) {
 807         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 808         if (localNextHopInfo != null) {
 809             long groupId;
 810             long localGroupId;
 811             final BigInteger dpnId = localNextHopInfo.getDpnId();
 812             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 813                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 814                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 815                 return dpnId;
 816             }
 817             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 818                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 819                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 820                 return dpnId;
 821             }
 822             if (!isVpnPresentInDpn(rd, dpnId)) {
 823                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 824                         vpnId, rd, dpnId.toString());
 825                 return BigInteger.ZERO;
 826             }
 827             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 828             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 829             String prefix = vrfEntry.getDestPrefix();
 830             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 831             //The loadbalancing group is created only if the extra route has multiple nexthops
 832             //to avoid loadbalancing the discovered routes
 833             if (vpnExtraRoutes != null && routes != null) {
 834                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 835                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 836                 } else {
 837                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 838                 }
 839                 if (vpnExtraRoutes.size() > 1) {
 840                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 841                             vpnExtraRoutes);
 842                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 843                 } else if (routes.getNexthopIpList().size() > 1) {
 844                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 845                             vpnExtraRoutes);
 846                     localGroupId = groupId;
 847                 } else {
 848                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 849                             prefix, gwMacAddress, jobKey);
 850                     localGroupId = groupId;
 851                 }
 852             } else {
 853                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 854                         gwMacAddress, jobKey);
 855                 localGroupId = groupId;
 856             }
 857             if (groupId == FibConstants.INVALID_GROUP_ID) {
 858                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 859                         prefix, rd, interfaceName, dpnId.toString());
 860                 return BigInteger.ZERO;
 861             }
 862             final List<InstructionInfo> instructions = Collections.singletonList(
 863                     new InstructionApplyActions(
 864                             Collections.singletonList(new ActionGroup(groupId))));
 865             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 866                     new InstructionApplyActions(
 867                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(groupId))));
 868             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 869             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 870             jobCoordinator.enqueueJob(jobKey, () -> {
 871                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 872                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 873                             NwConstants.ADD_FLOW, tx, null);
 874                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 875                             vpnName, rd)) {
 876                         optLabel.ifPresent(label -> {
 877                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 878                                 LOG.debug(
 879                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 880                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 881                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 882                                         nextHopAddressList);
 883                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 884                                         NwConstants.ADD_FLOW, tx);
 885                                 // If the extra-route is reachable from VMs attached to the same switch,
 886                                 // then the tunnel table can point to the load balancing group.
 887                                 // If it is reachable from VMs attached to different switches,
 888                                 // then it should be pointing to one of the local group in order to avoid looping.
 889                                 if (vrfEntry.getRoutePaths().size() == 1) {
 890                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 891                                 } else {
 892                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 893                                 }
 894                             } else {
 895                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 896                                                 + "route. LFib and Terminating table entries will not be created.",
 897                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 898                             }
 899                         });
 900                     }
 901                 }));
 902             });
 903             return dpnId;
 904         }
 905         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 906                 vpnName);
 907         return BigInteger.ZERO;
 908     }
 909
 910     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 911         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 912         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 913         if (dpnInVpn.isPresent()) {
 914             return true;
 915         }
 916         return false;
 917     }
 918
 919     private LabelRouteInfo getLabelRouteInfo(Long label) {
 920         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 921             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 922         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 923         if (opResult.isPresent()) {
 924             return opResult.get();
 925         }
 926         return null;
 927     }
 928
 929     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 930         if (lri == null) {
 931             return true;
 932         }
 933
 934         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 935         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 936             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 937
 938         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 939             ? lri.getVpnInstanceList() : new ArrayList<>();
 940         if (vpnInstancesList.contains(vpnInstanceName)) {
 941             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 942             vpnInstancesList.remove(vpnInstanceName);
 943         }
 944         if (vpnInstancesList.isEmpty()) {
 945             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 946             if (tx != null) {
 947                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 948             } else {
 949                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 950             }
 951             return true;
 952         } else {
 953             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 954             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 955             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 956         }
 957         return false;
 958     }
 959
 960     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 961                                       WriteTransaction tx) {
 962         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 963
 964         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 965
 966         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 967             dpId, label, groupId);
 968     }
 969
 970     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 971                                                 WriteTransaction tx) {
 972         List<MatchInfo> mkMatches = new ArrayList<>();
 973
 974         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 975             destDpId, label, actionsInfos);
 976
 977         // Matching metadata
 978         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 979         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 980
 981         List<InstructionInfo> mkInstructions = new ArrayList<>();
 982         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 983
 984         FlowEntity terminatingServiceTableFlowEntity =
 985             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 986             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 987                 String.format("%s:%d", "TST Flow Entry ", label),
 988             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 989
 990         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 991
 992         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 993
 994         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 995         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 996             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 997             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 998             .child(Flow.class, flowKey).build();
 999         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
1000                 WriteTransaction.CREATE_MISSING_PARENTS);
1001     }
1002
1003     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1004         FlowEntity flowEntity;
1005         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1006         List<MatchInfo> mkMatches = new ArrayList<>();
1007         // Matching metadata
1008         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1009         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1010             NwConstants.INTERNAL_TUNNEL_TABLE,
1011             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1012             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1013             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1014         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1015         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1016         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1017             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1018             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1019
1020         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1021         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1022     }
1023
1024     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1025         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1026         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1027         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1028         boolean shouldUpdateNonEcmpLocalNextHop = true;
1029         if (localNextHopInfo == null) {
1030             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1031             if (usedRds.size() > 1) {
1032                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1033                         vrfEntry.getDestPrefix(), vpnName, rd);
1034                 return returnLocalDpnId;
1035             }
1036             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1037             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1038             //in the vpn
1039             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1040                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1041             if (extraRouteOptional.isPresent()) {
1042                 Routes extraRoute = extraRouteOptional.get();
1043                 String ipPrefix;
1044                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1045                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1046                 } else {
1047                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1048                 }
1049                 if (extraRoute.getNexthopIpList().size() > 1) {
1050                     shouldUpdateNonEcmpLocalNextHop = false;
1051                 }
1052                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1053                 if (localNextHopInfo != null) {
1054                     String localNextHopIP = localNextHopInfo.getIpAddress();
1055                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1056                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1057                     if (!dpnId.equals(BigInteger.ZERO)) {
1058                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1059                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1060                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1061                                 /*remove*/ false);
1062                         returnLocalDpnId.add(dpnId);
1063                     }
1064                 } else {
1065                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1066                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1067                 }
1068             }
1069
1070             if (localNextHopInfo == null) {
1071                 /* Imported VRF entry */
1072                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1073                 if (optionalLabel.isPresent()) {
1074                     Long label = optionalLabel.get();
1075                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1076                     LabelRouteInfo lri = getLabelRouteInfo(label);
1077                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1078                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1079                         prefixBuilder.setDpnId(lri.getDpnId());
1080                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1081                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1082                         if (!dpnId.equals(BigInteger.ZERO)) {
1083                             returnLocalDpnId.add(dpnId);
1084                         }
1085                     }
1086                 }
1087             }
1088
1089         } else {
1090             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1091             String localNextHopIP = localNextHopInfo.getIpAddress();
1092             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1093                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1094             if (!dpnId.equals(BigInteger.ZERO)) {
1095                 returnLocalDpnId.add(dpnId);
1096             }
1097         }
1098
1099         return returnLocalDpnId;
1100     }
1101
1102     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1103                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1104                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1105         if (localNextHopInfo != null) {
1106             final BigInteger dpnId = localNextHopInfo.getDpnId();
1107             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1108                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1109                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1110                 return dpnId;
1111             }
1112             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1113                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1114                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1115                 return dpnId;
1116             }
1117
1118             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1119                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1120                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1121                             NwConstants.DEL_FLOW, tx, null);
1122                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1123                             vpnId, rd)) {
1124                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1125                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1126                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1127                                         NwConstants.DEL_FLOW, tx);
1128                                 removeTunnelTableEntry(dpnId, label, tx);
1129                             });
1130                         }
1131                     }
1132                 })));
1133             //TODO: verify below adjacency call need to be optimized (?)
1134             //In case of the removal of the extra route, the loadbalancing group is updated
1135             if (shouldUpdateNonEcmpLocalNextHop) {
1136                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1137             }
1138             return dpnId;
1139         }
1140         return BigInteger.ZERO;
1141     }
1142
1143     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1144             final VrfEntry vrfEntry, WriteTransaction tx) {
1145         if (tx == null) {
1146             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1147                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1148             }), LOG, "Error creating remote FIB entry");
1149             return;
1150         }
1151
1152         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1153         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1154                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1155
1156         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1157         if (adjacencyResults.isEmpty()) {
1158             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1159                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1160             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1161             return;
1162         }
1163
1164         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1165         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1166                 vpnName, usedRds, vrfEntry.getDestPrefix());
1167         // create loadbalancing groups for extra routes only when the extra route is present behind
1168         // multiple VMs
1169         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1170                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1171             List<InstructionInfo> instructions = new ArrayList<>();
1172             // Obtain the local routes for this particular dpn.
1173             java.util.Optional<Routes> routes = vpnExtraRoutes
1174                     .stream()
1175                     .filter(route -> {
1176                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1177                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1178                         if (prefixToInterface == null) {
1179                             return false;
1180                         }
1181                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1182                     }).findFirst();
1183             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1184                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1185             if (groupId == FibConstants.INVALID_GROUP_ID) {
1186                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1187                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1188                 return;
1189             }
1190             List<ActionInfo> actionInfos =
1191                     Collections.singletonList(new ActionGroup(groupId));
1192             instructions.add(new InstructionApplyActions(actionInfos));
1193             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1194                     NwConstants.ADD_FLOW, tx, null);
1195         } else {
1196             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1197         }
1198
1199         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1200     }
1201
1202     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1203     /* Get interface info from prefix to interface mapping;
1204         Use the interface info to get the corresponding vpn interface op DS entry,
1205         remove the adjacency corresponding to this fib entry.
1206         If adjacency removed is the last adjacency, clean up the following:
1207          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1208          - prefix to interface entry
1209          - vpn interface op DS
1210      */
1211         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1212         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1213         if (prefixInfo == null) {
1214             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1215             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1216             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1217             if (extraRoute != null) {
1218                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1219                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1220                     if (nextHopIp != null) {
1221                         String ipPrefix;
1222                         if (isIpv4Address(nextHopIp)) {
1223                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1224                         } else {
1225                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1226                         }
1227                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1228                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1229                     }
1230                 }
1231             }
1232             if (prefixInfo == null) {
1233                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1234                 if (optionalLabel.isPresent()) {
1235                     Long label = optionalLabel.get();
1236                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1237                     LabelRouteInfo lri = getLabelRouteInfo(label);
1238                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1239                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1240                         prefixBuilder.setDpnId(lri.getDpnId());
1241                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1242                         prefixBuilder.setIpAddress(lri.getPrefix());
1243                         prefixInfo = prefixBuilder.build();
1244                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1245                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1246                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1247                     }
1248                 }
1249             }
1250         } else {
1251             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1252         }
1253     }
1254
1255     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1256                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1257
1258         if (prefixInfo == null) {
1259             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1260                     vrfEntry.getDestPrefix(), vpnId, rd);
1261             return; //Don't have any info for this prefix (shouldn't happen); need to return
1262         }
1263
1264         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1265             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1266                     vrfEntry.getDestPrefix(), vpnId, rd);
1267             return;
1268         }
1269
1270         String ifName = prefixInfo.getVpnInterfaceName();
1271         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1272             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1273     }
1274
1275     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1276         Prefixes prefixInfo;
1277         Long vpnId;
1278         String rd;
1279         VrfEntry vrfEntry;
1280         Routes extraRoute;
1281
1282         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1283                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1284             this.prefixInfo = prefixInfo;
1285             this.vpnId = vpnId;
1286             this.rd = rd;
1287             this.vrfEntry = vrfEntry;
1288             this.extraRoute = extraRoute;
1289         }
1290
1291         @Override
1292         public List<ListenableFuture<Void>> call() {
1293             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1294             // to call the respective helpers.
1295             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1296                 //First Cleanup LabelRouteInfo
1297                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1298                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1299                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1300                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1301                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1302                         synchronized (label.toString().intern()) {
1303                             LabelRouteInfo lri = getLabelRouteInfo(label);
1304                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1305                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1306                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1307                                         fibUtil.getVpnInstanceOpData(rd);
1308                                 String vpnInstanceName = "";
1309                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1310                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1311                                 }
1312                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1313                                 if (lriRemoved) {
1314                                     String parentRd = lri.getParentVpnRd();
1315                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1316                                             parentRd, vrfEntry.getDestPrefix()));
1317                                 }
1318                             } else {
1319                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1320                                         rd, vrfEntry.getDestPrefix()));
1321                             }
1322                         }
1323                     });
1324                 }
1325                 String ifName = prefixInfo.getVpnInterfaceName();
1326                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1327                 String vpnName = null;
1328
1329                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1330                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1331                     * */
1332                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1333                     if (vpnNameOpt.isPresent()) {
1334                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1335                     }
1336                 }
1337                 if (optVpnName.isPresent()) {
1338                     vpnName = optVpnName.get();
1339                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1340                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1341                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1342                     if (opVpnInterface.isPresent()) {
1343                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1344                         if (vpnId != associatedVpnId) {
1345                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1346                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1347                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1348                             return;
1349                         } else {
1350                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1351                                     vrfEntry.getDestPrefix(), associatedVpnId);
1352                         }
1353                     }
1354                 }
1355                 if (extraRoute != null) {
1356                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1357                     //Only one used Rd present in case of removal event
1358                     String usedRd = usedRds.get(0);
1359                     if (optVpnName.isPresent()) {
1360                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1361                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1362                                         vrfEntry.getDestPrefix()));
1363                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1364                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1365                     }
1366                 }
1367                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1368             }));
1369         }
1370     }
1371
1372     /**
1373      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1374      * Remove Adjacency from VPNInterfaceOpData.
1375      * if Adjacency != primary.
1376      * if Adjacency == primary , then mark it for deletion.
1377      * Remove entire VPNinterfaceOpData Entry.
1378      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1379      * @param vrfEntry - VrfEntry removed
1380      * @param ifName - Interface name from VRFentry
1381      * @param vpnName - VPN name of corresponding VRF
1382      * @param tx - ReadWrite Tx
1383      * @throws ReadFailedException - Exception thrown in case of read failed
1384      */
1385     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1386                                                           ReadWriteTransaction tx) throws ReadFailedException {
1387         InstanceIdentifier<Adjacency> adjacencyIid =
1388                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1389         Optional<Adjacency> adjacencyOptional = tx.read(LogicalDatastoreType.OPERATIONAL, adjacencyIid).checkedGet();
1390         if (adjacencyOptional.isPresent()) {
1391             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1392                 tx.delete(LogicalDatastoreType.OPERATIONAL,
1393                         FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1394             } else {
1395                 tx.merge(LogicalDatastoreType.OPERATIONAL, adjacencyIid,
1396                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1397             }
1398         }
1399
1400         Optional<AdjacenciesOp> optAdjacencies =
1401                 tx.read(LogicalDatastoreType.OPERATIONAL,
1402                         FibUtil.getAdjListPathOp(ifName, vpnName)).checkedGet();
1403
1404         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1405             return;
1406         }
1407
1408         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1409                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1410                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1411                         && adjacency.isMarkedForDeletion() != null
1412                         && adjacency.isMarkedForDeletion()
1413         )) {
1414             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1415             tx.delete(LogicalDatastoreType.OPERATIONAL,
1416                     FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1417         }
1418     }
1419
1420     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1421         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1422         final String rd = vrfTableKey.getRouteDistinguisher();
1423         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1424         if (vpnInstance == null) {
1425             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1426             return;
1427         }
1428         final Collection<VpnToDpnList> vpnToDpnList;
1429         if (vrfEntry.getParentVpnRd() != null
1430                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1431             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1432             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1433             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1434                     vpnInstance.getVpnToDpnList();
1435             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1436                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1437         } else {
1438             vpnToDpnList = vpnInstance.getVpnToDpnList();
1439         }
1440
1441         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1442         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1443         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1444         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1445         if (subnetRoute != null) {
1446             long elanTag = subnetRoute.getElantag();
1447             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1448                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1449             if (vpnToDpnList != null) {
1450                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1451                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1452                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1453
1454                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1455                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1456                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1457                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1458                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1459                             }
1460
1461                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1462                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1463                         }
1464                     })));
1465             }
1466             optionalLabel.ifPresent(label -> {
1467                 synchronized (label.toString().intern()) {
1468                     LabelRouteInfo lri = getLabelRouteInfo(label);
1469                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1470                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1471                                 fibUtil.getVpnInstanceOpData(rd);
1472                         String vpnInstanceName = "";
1473                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1474                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1475                         }
1476                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1477                         if (lriRemoved) {
1478                             String parentRd = lri.getParentVpnRd();
1479                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1480                                     parentRd, vrfEntry.getDestPrefix()));
1481                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1482                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1483                         }
1484                     } else {
1485                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1486                                 rd, vrfEntry.getDestPrefix()));
1487                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1488                                 label, rd, vrfEntry.getDestPrefix());
1489                     }
1490                 }
1491             });
1492             return;
1493         }
1494
1495         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1496             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1497         if (vpnToDpnList != null) {
1498             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1499                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1500             String jobKey;
1501             Optional<Routes> extraRouteOptional;
1502             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1503             if (usedRds != null && !usedRds.isEmpty()) {
1504                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1505                 if (usedRds.size() > 1) {
1506                     LOG.error("The extra route prefix is still present in some DPNs");
1507                     return ;
1508                 } else {
1509                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1510                     //is not present in any other DPN
1511                     extraRouteOptional = VpnExtraRouteHelper
1512                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1513                 }
1514             } else {
1515                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1516                 extraRouteOptional = Optional.absent();
1517             }
1518
1519             jobCoordinator.enqueueJob(jobKey,
1520                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1521                     if (localDpnIdList.size() <= 0) {
1522                         for (VpnToDpnList curDpn : vpnToDpnList) {
1523                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1524                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1525                         }
1526                     } else {
1527                         for (BigInteger localDpnId : localDpnIdList) {
1528                             for (VpnToDpnList curDpn : vpnToDpnList) {
1529                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1530                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1531                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1532                                 }
1533                             }
1534                         }
1535                     }
1536                 })), MAX_RETRIES);
1537         }
1538
1539         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1540         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1541         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1542
1543         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1544         // of the interVpnLink.
1545         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1546         if (optVpnUuid.isPresent()) {
1547             String vpnUuid = optVpnUuid.get();
1548             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1549                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1550                 if (optInterVpnLink.isPresent()) {
1551                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1552                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1553                         // This is route that points to the other endpoint of an InterVpnLink
1554                         // In that case, we should look for the FIB table pointing to
1555                         // LPortDispatcher table and remove it.
1556                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1557                     }
1558                 }
1559             });
1560         }
1561
1562     }
1563
1564     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1565                                     int addOrRemove, WriteTransaction tx) {
1566         if (tx == null) {
1567             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1568                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1569                 "Error making LFIB table entry");
1570             return;
1571         }
1572
1573         List<MatchInfo> matches = new ArrayList<>();
1574         matches.add(MatchEthernetType.MPLS_UNICAST);
1575         matches.add(new MatchMplsLabel(label));
1576
1577         // Install the flow entry in L3_LFIB_TABLE
1578         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1579
1580         FlowEntity flowEntity;
1581         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1582             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1583         Flow flow = flowEntity.getFlowBuilder().build();
1584         String flowId = flowEntity.getFlowId();
1585         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1586         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1587         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1588             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1589             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1590
1591         if (addOrRemove == NwConstants.ADD_FLOW) {
1592             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1593         } else {
1594             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1595         }
1596
1597         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1598             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1599     }
1600
1601     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1602                                     final FutureCallback<List<Void>> callback) {
1603         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1604         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1605             () -> {
1606                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1607                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1608                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1609                         LogicalDatastoreType.CONFIGURATION, id);
1610                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1611                 if (!vrfTable.isPresent()) {
1612                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1613                     if (callback != null) {
1614                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1615                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1616                     }
1617                     return futures;
1618                 }
1619                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1620                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1621                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1622                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1623                             if (subnetRoute != null) {
1624                                 long elanTag = subnetRoute.getElantag();
1625                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1626                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1627                                         tx);
1628                                 continue;
1629                             }
1630                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1631                             if (routerInt != null) {
1632                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1633                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1634                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1635                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1636                                         NwConstants.ADD_FLOW);
1637                                 continue;
1638                             }
1639                             //Handle local flow creation for imports
1640                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1641                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1642                                 if (optionalLabel.isPresent()) {
1643                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1644                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1645                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1646                                         if (lri.getDpnId().equals(dpnId)) {
1647                                             try {
1648                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1649                                                         vrfEntry.getDestPrefix());
1650                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1651                                             } catch (IllegalArgumentException ex) {
1652                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1653                                                         vrfEntry.getDestPrefix());
1654                                             }
1655                                             continue;
1656                                         }
1657                                     }
1658                                 }
1659                             }
1660                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1661                                     vrfEntry, dpnId);
1662                             if (shouldCreateRemoteFibEntry) {
1663                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1664                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1665                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1666                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1667                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1668                                 } else {
1669                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1670                                             vrfEntry, tx);
1671                                 }
1672                             }
1673                         }
1674                     }));
1675                     if (callback != null) {
1676                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1677                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1678                     }
1679                 }
1680                 return futures;
1681             });
1682     }
1683
1684     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1685                                             final String localNextHopIp, final String remoteNextHopIp) {
1686         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1687             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1688         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1689         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1690         List<SubTransaction> txnObjects =  new ArrayList<>();
1691         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1692         if (vrfTable.isPresent()) {
1693             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1694                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1695                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1696                         vrfTable.get().getVrfEntry().stream()
1697                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1698                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1699                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1700                     }
1701                 })));
1702         }
1703     }
1704
1705     public void manageRemoteRouteOnDPN(final boolean action,
1706                                        final BigInteger localDpnId,
1707                                        final long vpnId,
1708                                        final String rd,
1709                                        final String destPrefix,
1710                                        final String destTepIp,
1711                                        final long label) {
1712         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1713
1714         if (vpnInstance == null) {
1715             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1716             return;
1717         }
1718
1719         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1720             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1721                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1722                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1723                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1724                     if (vrfEntry == null) {
1725                         return;
1726                     }
1727                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1728                             action, localDpnId, vpnId, rd, destPrefix);
1729                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1730                     VrfEntry modVrfEntry;
1731                     if (routePathList == null || routePathList.isEmpty()) {
1732                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1733                                 Collections.singletonList(destTepIp),
1734                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1735                     } else {
1736                         modVrfEntry = vrfEntry;
1737                     }
1738
1739                     if (action) {
1740                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1741                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1742                                 modVrfEntry, tx);
1743                     } else {
1744                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1745                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1746                                 vrfEntry.getDestPrefix());
1747                         if (usedRds.size() > 1) {
1748                             LOG.debug("The extra route prefix is still present in some DPNs");
1749                             return;
1750                         }
1751                         //Is this fib route an extra route? If yes, get the nexthop which would be
1752                         //an adjacency in the vpn
1753                         Optional<Routes> extraRouteOptional = Optional.absent();
1754                         if (usedRds.size() != 0) {
1755                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1756                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1757                                     usedRds.get(0), vrfEntry.getDestPrefix());
1758                         }
1759                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1760                                 extraRouteOptional, tx);
1761                     }
1762                 }
1763             })));
1764     }
1765
1766     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1767                                  final FutureCallback<List<Void>> callback) {
1768         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1769         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1770             () -> {
1771                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1772                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1773                 List<SubTransaction> txnObjects =  new ArrayList<>();
1774                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1775                         LogicalDatastoreType.CONFIGURATION, id);
1776                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1777                 if (vrfTable.isPresent()) {
1778                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1779                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1780                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1781                                 /* Handle subnet routes here */
1782                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1783                                 if (subnetRoute != null) {
1784                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1785                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1786                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1787                                             NwConstants.DEL_FLOW, tx, null);
1788                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1789                                     if (routePaths != null) {
1790                                         for (RoutePaths routePath : routePaths) {
1791                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1792                                                     DEFAULT_FIB_FLOW_PRIORITY,
1793                                                     NwConstants.DEL_FLOW, tx);
1794                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1795                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1796                                                     vrfEntry.getDestPrefix());
1797                                         }
1798                                     }
1799                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1800                                             NwConstants.DEL_FLOW, tx);
1801                                     continue;
1802                                 }
1803                                 // ping responder for router interfaces
1804                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1805                                 if (routerInt != null) {
1806                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1807                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1808                                             routerInt.getMacAddress());
1809                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1810                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1811                                             NwConstants.DEL_FLOW);
1812                                     continue;
1813                                 }
1814
1815                                 //Handle local flow deletion for imports
1816                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1817                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1818                                     if (optionalLabel.isPresent()) {
1819                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1820                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1821                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1822                                                 lri) && lri.getDpnId().equals(dpnId)) {
1823                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1824                                         }
1825                                     }
1826                                 }
1827
1828                                 // Passing null as we don't know the dpn
1829                                 // to which prefix is attached at this point
1830                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1831                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1832                                 String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1833                                 Optional<Routes> extraRouteOptional;
1834                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1835                                 //an adjacency in the vpn
1836                                 if (usedRds != null && !usedRds.isEmpty()) {
1837                                     if (usedRds.size() > 1) {
1838                                         LOG.error("The extra route prefix is still present in some DPNs");
1839                                         return;
1840                                     } else {
1841                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1842                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1843
1844                                     }
1845                                 } else {
1846                                     extraRouteOptional = Optional.absent();
1847                                 }
1848                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1849                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1850                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1851                                 } else {
1852                                     baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(),
1853                                             vrfEntry, extraRouteOptional, tx);
1854                                 }
1855                             }
1856                         }));
1857                     }
1858                     if (callback != null) {
1859                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1860                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1861                     }
1862                 } else {
1863                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1864                 }
1865                 return futures;
1866             });
1867
1868     }
1869
1870     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1871                                            final String localNextHopIp, final String remoteNextHopIp) {
1872         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1873                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1874             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1875         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1876         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1877         List<SubTransaction> txnObjects =  new ArrayList<>();
1878         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1879         if (vrfTable.isPresent()) {
1880             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1881                 () -> {
1882                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1883                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1884                             tx -> vrfTable.get().getVrfEntry().stream()
1885                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1886                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1887                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1888                     }
1889                 });
1890         }
1891     }
1892
1893     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1894         InstanceIdentifierBuilder<VrfTables> idBuilder =
1895             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1896         return idBuilder.build();
1897     }
1898
1899     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1900         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1901                 .FLOWID_SEPARATOR + nextHop;
1902     }
1903
1904     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1905         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1906                 + tableMiss + FLOWID_PREFIX;
1907     }
1908
1909     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1910         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1911             .child(VrfTables.class, new VrfTablesKey(rd))
1912             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1913         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1914         if (vrfEntry.isPresent()) {
1915             return vrfEntry.get();
1916         }
1917         return null;
1918     }
1919
1920     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1921                                              final String vpnName,
1922                                              final VrfEntry vrfEntry) {
1923         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1924
1925         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1926         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1927
1928         if (targetDpns.isEmpty()) {
1929             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1930             return;
1931         }
1932
1933         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1934         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1935
1936         // delete from FIB
1937         //
1938         optNextHop.ifPresent(nextHop -> {
1939             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1940             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1941             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1942                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1943
1944             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1945             for (BigInteger dpId : targetDpns) {
1946                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1947                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1948
1949                 mdsalManager.removeFlow(dpId, flow);
1950             }
1951         });
1952
1953         // delete from LFIB
1954         //
1955         optLabel.ifPresent(label -> {
1956             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1957
1958             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1959                 for (BigInteger dpId : targetDpns) {
1960                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1961                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1962                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1963                             tx);
1964                 }
1965             }), LOG, "Error removing flows");
1966         });
1967     }
1968
1969     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1970             List<String> nextHopAddressList, LabelRouteInfo lri) {
1971         return lri != null && lri.getPrefix().equals(prefix)
1972                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1973     }
1974
1975     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1976         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1977             return true;
1978         }
1979
1980         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1981         if (prefix != null) {
1982             BigInteger prefixDpnId = prefix.getDpnId();
1983             if (dpnId.equals(prefixDpnId)) {
1984                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1985                         vrfEntry, dpnId);
1986                 return false;
1987             }
1988         }
1989         return true;
1990     }
1991 }