fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  41 import org.opendaylight.genius.mdsalutil.ActionInfo;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  44 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  45 import org.opendaylight.genius.mdsalutil.MatchInfo;
  46 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  51 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  52 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  53 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  54 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  55 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  56 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  60 import org.opendaylight.genius.utils.ServiceIndex;
  61 import org.opendaylight.genius.utils.batching.SubTransaction;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  65 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  66 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  69 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  70 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  71 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 105 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 107 import org.slf4j.Logger;
 108 import org.slf4j.LoggerFactory;
 109
 110
 111 @Singleton
 112 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 115     private static final String FLOWID_PREFIX = "L3.";
 116     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 117     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 118     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 119     private static final int LFIB_INTERVPN_PRIORITY = 15;
 120     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 121     private static final int MAX_RETRIES = 3;
 122     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 123
 124     private final DataBroker dataBroker;
 125     private final ManagedNewTransactionRunner txRunner;
 126     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 127     private final IMdsalApiManager mdsalManager;
 128     private final NexthopManager nextHopManager;
 129     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 130     private final BaseVrfEntryHandler baseVrfEntryHandler;
 131     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 132     private final JobCoordinator jobCoordinator;
 133     private final IElanService elanManager;
 134     private final FibUtil fibUtil;
 135     private final InterVpnLinkCache interVpnLinkCache;
 136     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 137
 138     @Inject
 139     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 140                             final NexthopManager nexthopManager,
 141                             final IElanService elanManager,
 142                             final BaseVrfEntryHandler vrfEntryHandler,
 143                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 144                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 145                             final JobCoordinator jobCoordinator,
 146                             final FibUtil fibUtil,
 147                             final InterVpnLinkCache interVpnLinkCache) {
 148         super(VrfEntry.class, VrfEntryListener.class);
 149         this.dataBroker = dataBroker;
 150         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 151         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 152         this.mdsalManager = mdsalApiManager;
 153         this.nextHopManager = nexthopManager;
 154         this.elanManager = elanManager;
 155         this.baseVrfEntryHandler = vrfEntryHandler;
 156         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 157         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 158         this.jobCoordinator = jobCoordinator;
 159         this.fibUtil = fibUtil;
 160         this.interVpnLinkCache = interVpnLinkCache;
 161     }
 162
 163     @Override
 164     @PostConstruct
 165     public void init() {
 166         LOG.info("{} init", getClass().getSimpleName());
 167         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 168     }
 169
 170     @Override
 171     @SuppressWarnings("checkstyle:IllegalCatch")
 172     public void close() {
 173         closeables.forEach(c -> {
 174             try {
 175                 c.close();
 176             } catch (Exception e) {
 177                 LOG.warn("Error closing {}", c, e);
 178             }
 179         });
 180     }
 181
 182     @Override
 183     protected VrfEntryListener getDataTreeChangeListener() {
 184         return VrfEntryListener.this;
 185     }
 186
 187     @Override
 188     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 189         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 190     }
 191
 192     @Override
 193     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 194         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 195         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 196         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 197                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 198         addFibEntries(identifier, vrfEntry, rd);
 199         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 200                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 201     }
 202
 203     //This method is temporary. Eventually Factory design pattern will be used to get
 204     // right VrfEntryhandle and invoke its methods.
 205     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 206         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 207             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 208             return;
 209         }
 210         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 211             LOG.info("EVPN flows need to be programmed.");
 212             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 213                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 214             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 215             closeables.add(evpnVrfEntryHandler);
 216             return;
 217         }
 218         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 219         if (routerInt != null) {
 220             // ping responder for router interfaces
 221             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 222             return;
 223         }
 224         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 225             createFibEntries(identifier, vrfEntry);
 226             return;
 227         }
 228     }
 229
 230     @Override
 231     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 232         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 233         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 234         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 235                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 236         removeFibEntries(identifier, vrfEntry, rd);
 237         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 238             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 239     }
 240
 241     //This method is temporary. Eventually Factory design pattern will be used to get
 242     // right VrfEntryhandle and invoke its methods.
 243     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 244         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 245             LOG.info("EVPN flows to be deleted");
 246             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 247                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 248             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 249             closeables.add(evpnVrfEntryHandler);
 250             return;
 251         }
 252         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 253         if (routerInt != null) {
 254             // ping responder for router interfaces
 255             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 256             return;
 257         }
 258         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 259             deleteFibEntries(identifier, vrfEntry);
 260             return;
 261         }
 262         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 263             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 264             return;
 265         }
 266     }
 267
 268     @Override
 269     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 270     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 271     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 272     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 273         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 274         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 275         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 276                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 277         // Handle BGP Routes first
 278         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 279             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 280             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 281                     rd, update.getDestPrefix(), update.getRoutePaths());
 282             return;
 283         }
 284
 285         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 286             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 287             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 288             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 289
 290             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 291             //updated vrf entry route path is null. This can happen during tunnel events.
 292             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 293             List<String> usedRds = new ArrayList<>();
 294             if (optVpnInstance.isPresent()) {
 295                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 296                         update.getDestPrefix());
 297             }
 298             // If original VRF Entry had nexthop null , but update VRF Entry
 299             // has nexthop , route needs to be created on remote Dpns
 300             if (originalRoutePath == null || originalRoutePath.isEmpty()
 301                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 302                 // TODO(vivek): Though ugly, Not handling this code now, as each
 303                 // tep add event will invoke flow addition
 304                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 305                         + " This event is IGNORED here.", update.getDestPrefix());
 306                 return;
 307             }
 308
 309             // If original VRF Entry had valid nexthop , but update VRF Entry
 310             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 311             if (updateRoutePath == null || updateRoutePath.isEmpty()
 312                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 313                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 314                         + "This event is IGNORED here.", update.getDestPrefix());
 315                 return;
 316             }
 317             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 318             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 319             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 320             ListenableFuture<Void> future =
 321                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> nextHopsRemoved.parallelStream()
 322                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 323                                     tx, nextHopRemoved, rd, update.getDestPrefix())));
 324             Futures.addCallback(future, new FutureCallback<Void>() {
 325                 @Override
 326                 public void onSuccess(Void result) {
 327                     createFibEntries(identifier, update);
 328                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 329                             rd, update.getDestPrefix(), update.getRoutePaths());
 330                 }
 331
 332                 @Override
 333                 public void onFailure(Throwable throwable) {
 334                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 335                             update, throwable);
 336                 }
 337             }, MoreExecutors.directExecutor());
 338         }
 339
 340         //Handle all other routes only on a cluster reboot
 341         if (original.equals(update)) {
 342             //Reboot use-case
 343             createFibEntries(identifier, update);
 344             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 345                     rd, update.getDestPrefix(), update.getRoutePaths());
 346             return;
 347         }
 348
 349         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 350                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 351     }
 352
 353     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 354         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 355         List<SubTransaction> txnObjects =  new ArrayList<>();
 356         final VpnInstanceOpDataEntry vpnInstance =
 357                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 358         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 359         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 360                 + " has null vpnId!");
 361         final Collection<VpnToDpnList> vpnToDpnList;
 362         if (vrfEntry.getParentVpnRd() != null
 363                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 364             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 365             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 366             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 367                 vpnInstance.getVpnToDpnList();
 368             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 369                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 370         } else {
 371             vpnToDpnList = vpnInstance.getVpnToDpnList();
 372         }
 373         final Long vpnId = vpnInstance.getVpnId();
 374         final String rd = vrfTableKey.getRouteDistinguisher();
 375         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 376         if (subnetRoute != null) {
 377             final long elanTag = subnetRoute.getElantag();
 378             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 379                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 380             if (vpnToDpnList != null) {
 381                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 382                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 383                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 384                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 385                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 386                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 387                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 388                             }
 389                         }
 390                     })));
 391             }
 392             return;
 393         }
 394
 395         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 396         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 397             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 398                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 399                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 400                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 401                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 402                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 403                                     try {
 404                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 405                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 406                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 407                                                     txnObjects);
 408                                         } else {
 409                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 410                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 411                                         }
 412                                     } catch (NullPointerException e) {
 413                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 414                                                 vrfEntry.getDestPrefix(), e);
 415                                     }
 416                                 }
 417                             }
 418                         }
 419                     }
 420                 })), MAX_RETRIES);
 421         }
 422
 423         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 424         if (optVpnUuid.isPresent()) {
 425             String vpnUuid = optVpnUuid.get();
 426             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 427             if (interVpnLink != null) {
 428                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 429                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 430                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 431                         // This is an static route that points to the other endpoint of an InterVpnLink
 432                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 433                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 434                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
 435                     }
 436                 });
 437             }
 438         }
 439     }
 440
 441     void refreshFibTables(String rd, String prefix) {
 442         InstanceIdentifier<VrfEntry> vrfEntryId =
 443                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 444                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 445         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 446         if (vrfEntry.isPresent()) {
 447             createFibEntries(vrfEntryId, vrfEntry.get());
 448         }
 449     }
 450
 451     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 452         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 453         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 454         prefixBuilder.setDpnId(lri.getDpnId());
 455         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 456         prefixBuilder.setIpAddress(lri.getPrefix());
 457         // Increment the refCount here
 458         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 459             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 460         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 461         if (!isPresentInList) {
 462             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 463             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 464             vpnInstanceNames.add(vpnInstanceName);
 465             builder.setVpnInstanceList(vpnInstanceNames);
 466             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 467         } else {
 468             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 469         }
 470         return prefixBuilder.build();
 471     }
 472
 473     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 474                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 475         if (tx == null) {
 476             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 477                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 478                 "Error installing subnet route in FIB");
 479             return;
 480         }
 481         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 482             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 483             synchronized (label.toString().intern()) {
 484                 LabelRouteInfo lri = getLabelRouteInfo(label);
 485                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 486
 487                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 488                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 489                                 fibUtil.getVpnInstanceOpData(rd);
 490                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 491                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 492                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 493                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 494                             }
 495                         }
 496                     }
 497                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 498                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 499                 }
 500             }
 501         });
 502         final List<InstructionInfo> instructions = new ArrayList<>();
 503         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 504             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 505         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 506         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 507         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 508                 NwConstants.ADD_FLOW, tx, null);
 509
 510         if (vrfEntry.getRoutePaths() != null) {
 511             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 512                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 513                     List<ActionInfo> actionsInfos = new ArrayList<>();
 514                     // reinitialize instructions list for LFIB Table
 515                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 516
 517                     actionsInfos.add(new ActionPopMpls());
 518                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 519                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 520                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 521                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 522
 523                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 524                             NwConstants.ADD_FLOW, tx);
 525                 }
 526             }
 527         }
 528     }
 529
 530     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 531                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 532         List<MatchInfo> matches = new ArrayList<>();
 533
 534         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 535                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 536         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 537         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 538         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 539
 540         InetAddress destPrefix;
 541         try {
 542             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 543         } catch (UnknownHostException e) {
 544             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 545                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 546             return;
 547         }
 548
 549         // Match on VpnId and SubnetBroadCast IP address
 550         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 551         matches.add(MatchEthernetType.IPV4);
 552
 553         if (prefixLength != 0) {
 554             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 555         }
 556
 557         //Action is to drop the packet
 558         List<InstructionInfo> dropInstructions = new ArrayList<>();
 559         List<ActionInfo> actionsInfos = new ArrayList<>();
 560         actionsInfos.add(new ActionDrop());
 561         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 562
 563         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 564         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 565         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 566                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 567
 568         Flow flow = flowEntity.getFlowBuilder().build();
 569         String flowId = flowEntity.getFlowId();
 570         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 571         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 572
 573         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 574                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 575                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 576
 577         if (addOrRemove == NwConstants.ADD_FLOW) {
 578             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 579         } else {
 580             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 581         }
 582     }
 583
 584     /*
 585      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 586      * LportDispatcher table (via table 80)
 587      */
 588     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 589                                             final VrfEntry vrfEntry) {
 590         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 591         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 592         // packet is commuted from Vpn2 to Vpn1.
 593         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 594         if (!interVpnLink.isActive()) {
 595             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 596                      interVpnLinkName, vrfEntry.getDestPrefix());
 597             return;
 598         }
 599
 600         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 601         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 602         if (!optLportTag.isPresent()) {
 603             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 604             return;
 605         }
 606
 607         Long lportTag = optLportTag.get();
 608         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 609         if (label == null) {
 610             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 611                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 612             return;
 613         }
 614         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 615         List<InstructionInfo> instructions = Arrays.asList(
 616             new InstructionApplyActions(actionsInfos),
 617             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 618                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 619                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 620                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 621             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 622         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 623
 624         for (BigInteger dpId : targetDpns) {
 625             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 626                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 627
 628             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 629                                /*writeTx*/null);
 630         }
 631     }
 632
 633
 634     /*
 635      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 636      */
 637     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 638                                                final VrfEntry vrfEntry, long vpnTag) {
 639         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 640         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 641             && vrfEntry.getRoutePaths().size() == 1);
 642         String destination = vrfEntry.getDestPrefix();
 643         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 644         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 645
 646         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 647         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 648         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 649         if (interVpnLink.getState().or(State.Error) != State.Active) {
 650             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 651                 destination, nextHop, interVpnLinkName);
 652             return;
 653         }
 654
 655         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 656         if (!optOtherEndpointLportTag.isPresent()) {
 657             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 658                 vpnUuid, interVpnLinkName);
 659             return;
 660         }
 661
 662         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 663         if (targetDpns.isEmpty()) {
 664             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 665                 vpnUuid, interVpnLinkName);
 666             return;
 667         }
 668
 669         String[] values = destination.split("/");
 670         String destPrefixIpAddress = values[0];
 671         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 672
 673         List<MatchInfo> matches = new ArrayList<>();
 674         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 675         matches.add(MatchEthernetType.IPV4);
 676
 677         if (prefixLength != 0) {
 678             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 679         }
 680
 681         List<Instruction> instructions =
 682             Arrays.asList(new InstructionWriteMetadata(
 683                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 684                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 685                             .L3VPN_SERVICE_INDEX)),
 686                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 687                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 688
 689         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 690         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 691         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 692             COOKIE_VM_FIB_TABLE, matches, instructions);
 693
 694         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 695             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 696
 697         for (BigInteger dpId : targetDpns) {
 698
 699             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 700                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 701                 dpId, interVpnLink.getInterVpnLinkName());
 702
 703             mdsalManager.installFlow(dpId, flowEntity);
 704         }
 705     }
 706
 707     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 708         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 709         String localNextHopIP = vrfEntry.getDestPrefix();
 710         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 711         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 712         if (localNextHopInfo == null) {
 713             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 714             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 715                     vpnName, usedRds, localNextHopIP);
 716             boolean localNextHopSeen = false;
 717             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 718             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 719                 String ipPrefix;
 720                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 721                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 722                 } else {
 723                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 724                 }
 725                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 726                     ipPrefix);
 727                 if (localNextHopInfoLocal != null) {
 728                     localNextHopSeen = true;
 729                     BigInteger dpnId =
 730                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 731                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes);
 732                     returnLocalDpnId.add(dpnId);
 733                 }
 734             }
 735             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 736                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 737                 if (optionalLabel.isPresent()) {
 738                     Long label = optionalLabel.get();
 739                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 740                     synchronized (label.toString().intern()) {
 741                         LabelRouteInfo lri = getLabelRouteInfo(label);
 742                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 743                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 744                                     fibUtil.getVpnInstanceOpData(rd);
 745                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 746                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 747                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 748                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 749                                     localNextHopIP = lri.getPrefix();
 750                                 } else {
 751                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 752                                     localNextHopIP = lri.getPrefix();
 753                                 }
 754                             }
 755                             if (localNextHopInfo != null) {
 756                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 757                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 758                                 if (vpnExtraRoutes.isEmpty()) {
 759                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 760                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes);
 761                                     returnLocalDpnId.add(dpnId);
 762                                 } else {
 763                                     for (Routes extraRoutes : vpnExtraRoutes) {
 764                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 765                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes);
 766                                         returnLocalDpnId.add(dpnId);
 767                                     }
 768                                 }
 769                             }
 770                         }
 771                     }
 772                 }
 773             }
 774             if (returnLocalDpnId.isEmpty()) {
 775                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 776             }
 777         } else {
 778             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 779                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null);
 780             if (dpnId != null) {
 781                 returnLocalDpnId.add(dpnId);
 782             }
 783         }
 784         return returnLocalDpnId;
 785     }
 786
 787     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 788                                                 final Long vpnId, final String rd,
 789                                                 final VrfEntry vrfEntry,
 790                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 791         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 792         if (localNextHopInfo != null) {
 793             long groupId;
 794             long localGroupId;
 795             final BigInteger dpnId = localNextHopInfo.getDpnId();
 796             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 797                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 798                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 799                 return dpnId;
 800             }
 801             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 802                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 803                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 804                 return dpnId;
 805             }
 806             if (!isVpnPresentInDpn(rd, dpnId)) {
 807                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 808                         vpnId, rd, dpnId.toString());
 809                 return BigInteger.ZERO;
 810             }
 811             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 812             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 813             String prefix = vrfEntry.getDestPrefix();
 814             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 815             //The loadbalancing group is created only if the extra route has multiple nexthops
 816             //to avoid loadbalancing the discovered routes
 817             if (vpnExtraRoutes != null && routes != null) {
 818                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 819                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 820                 } else {
 821                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 822                 }
 823                 if (vpnExtraRoutes.size() > 1) {
 824                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 825                             vpnExtraRoutes);
 826                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 827                 } else if (routes.getNexthopIpList().size() > 1) {
 828                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 829                             vpnExtraRoutes);
 830                     localGroupId = groupId;
 831                 } else {
 832                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 833                             prefix, gwMacAddress, jobKey);
 834                     localGroupId = groupId;
 835                 }
 836             } else {
 837                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 838                         gwMacAddress, jobKey);
 839                 localGroupId = groupId;
 840             }
 841             if (groupId == FibConstants.INVALID_GROUP_ID) {
 842                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 843                         prefix, rd, interfaceName, dpnId.toString());
 844                 return BigInteger.ZERO;
 845             }
 846             final List<InstructionInfo> instructions = Collections.singletonList(
 847                     new InstructionApplyActions(
 848                             Collections.singletonList(new ActionGroup(groupId))));
 849             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 850                     new InstructionApplyActions(
 851                             Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 852             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 853             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 854             jobCoordinator.enqueueJob(jobKey, () -> {
 855                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 856                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 857                             NwConstants.ADD_FLOW, tx, null);
 858                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 859                             vpnName, rd)) {
 860                         optLabel.ifPresent(label -> {
 861                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 862                                 LOG.debug(
 863                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 864                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 865                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 866                                         nextHopAddressList);
 867                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 868                                         NwConstants.ADD_FLOW, tx);
 869                                 // If the extra-route is reachable from VMs attached to the same switch,
 870                                 // then the tunnel table can point to the load balancing group.
 871                                 // If it is reachable from VMs attached to different switches,
 872                                 // then it should be pointing to one of the local group in order to avoid looping.
 873                                 if (vrfEntry.getRoutePaths().size() == 1) {
 874                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 875                                 } else {
 876                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 877                                 }
 878                             } else {
 879                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 880                                                 + "route. LFib and Terminating table entries will not be created.",
 881                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 882                             }
 883                         });
 884                     }
 885                 }));
 886             });
 887             return dpnId;
 888         }
 889         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 890                 vpnName);
 891         return BigInteger.ZERO;
 892     }
 893
 894     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 895         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 896         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 897         if (dpnInVpn.isPresent()) {
 898             return true;
 899         }
 900         return false;
 901     }
 902
 903     private LabelRouteInfo getLabelRouteInfo(Long label) {
 904         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 905             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 906         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 907         if (opResult.isPresent()) {
 908             return opResult.get();
 909         }
 910         return null;
 911     }
 912
 913     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 914         if (lri == null) {
 915             return true;
 916         }
 917
 918         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 919         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 920             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 921
 922         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 923             ? lri.getVpnInstanceList() : new ArrayList<>();
 924         if (vpnInstancesList.contains(vpnInstanceName)) {
 925             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 926             vpnInstancesList.remove(vpnInstanceName);
 927         }
 928         if (vpnInstancesList.isEmpty()) {
 929             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 930             if (tx != null) {
 931                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 932             } else {
 933                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 934             }
 935             return true;
 936         } else {
 937             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 938             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 939             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 940         }
 941         return false;
 942     }
 943
 944     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 945                                       WriteTransaction tx) {
 946         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 947
 948         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 949
 950         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 951             dpId, label, groupId);
 952     }
 953
 954     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 955                                                 WriteTransaction tx) {
 956         List<MatchInfo> mkMatches = new ArrayList<>();
 957
 958         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 959             destDpId, label, actionsInfos);
 960
 961         // Matching metadata
 962         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 963         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 964
 965         List<InstructionInfo> mkInstructions = new ArrayList<>();
 966         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 967
 968         FlowEntity terminatingServiceTableFlowEntity =
 969             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 970             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 971                 String.format("%s:%d", "TST Flow Entry ", label),
 972             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 973
 974         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 975
 976         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 977
 978         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 979         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 980             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 981             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 982             .child(Flow.class, flowKey).build();
 983         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
 984                 WriteTransaction.CREATE_MISSING_PARENTS);
 985     }
 986
 987     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 988         FlowEntity flowEntity;
 989         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
 990         List<MatchInfo> mkMatches = new ArrayList<>();
 991         // Matching metadata
 992         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 993         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 994             NwConstants.INTERNAL_TUNNEL_TABLE,
 995             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
 996             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
 997             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 998         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
 999         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1000         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1001             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1002             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1003
1004         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1005         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1006     }
1007
1008     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1009         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1010         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1011         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1012         boolean shouldUpdateNonEcmpLocalNextHop = true;
1013         if (localNextHopInfo == null) {
1014             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1015             if (usedRds.size() > 1) {
1016                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1017                         vrfEntry.getDestPrefix(), vpnName, rd);
1018                 return returnLocalDpnId;
1019             }
1020             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1021             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1022             //in the vpn
1023             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1024                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1025             if (extraRouteOptional.isPresent()) {
1026                 Routes extraRoute = extraRouteOptional.get();
1027                 String ipPrefix;
1028                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1029                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1030                 } else {
1031                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1032                 }
1033                 if (extraRoute.getNexthopIpList().size() > 1) {
1034                     shouldUpdateNonEcmpLocalNextHop = false;
1035                 }
1036                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1037                 if (localNextHopInfo != null) {
1038                     String localNextHopIP = localNextHopInfo.getIpAddress();
1039                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1040                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1041                     if (!dpnId.equals(BigInteger.ZERO)) {
1042                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1043                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1044                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1045                                 /*remove*/ false);
1046                         returnLocalDpnId.add(dpnId);
1047                     }
1048                 } else {
1049                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1050                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1051                 }
1052             }
1053
1054             if (localNextHopInfo == null) {
1055                 /* Imported VRF entry */
1056                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1057                 if (optionalLabel.isPresent()) {
1058                     Long label = optionalLabel.get();
1059                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1060                     LabelRouteInfo lri = getLabelRouteInfo(label);
1061                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1062                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1063                         prefixBuilder.setDpnId(lri.getDpnId());
1064                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1065                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1066                         if (!dpnId.equals(BigInteger.ZERO)) {
1067                             returnLocalDpnId.add(dpnId);
1068                         }
1069                     }
1070                 }
1071             }
1072
1073         } else {
1074             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1075             String localNextHopIP = localNextHopInfo.getIpAddress();
1076             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1077                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1078             if (!dpnId.equals(BigInteger.ZERO)) {
1079                 returnLocalDpnId.add(dpnId);
1080             }
1081         }
1082
1083         return returnLocalDpnId;
1084     }
1085
1086     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1087                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1088                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1089         if (localNextHopInfo != null) {
1090             final BigInteger dpnId = localNextHopInfo.getDpnId();
1091             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1092                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1093                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1094                 return dpnId;
1095             }
1096             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1097                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1098                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1099                 return dpnId;
1100             }
1101
1102             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1103                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1104                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1105                             NwConstants.DEL_FLOW, tx, null);
1106                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1107                             vpnId, rd)) {
1108                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1109                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1110                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1111                                         NwConstants.DEL_FLOW, tx);
1112                                 removeTunnelTableEntry(dpnId, label, tx);
1113                             });
1114                         }
1115                     }
1116                 })));
1117             //TODO: verify below adjacency call need to be optimized (?)
1118             //In case of the removal of the extra route, the loadbalancing group is updated
1119             if (shouldUpdateNonEcmpLocalNextHop) {
1120                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1121             }
1122             return dpnId;
1123         }
1124         return BigInteger.ZERO;
1125     }
1126
1127     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1128             final VrfEntry vrfEntry, WriteTransaction tx) {
1129         if (tx == null) {
1130             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1131                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1132             }), LOG, "Error creating remote FIB entry");
1133             return;
1134         }
1135
1136         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1137         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1138                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1139
1140         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1141         if (adjacencyResults.isEmpty()) {
1142             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1143                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1144             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1145             return;
1146         }
1147
1148         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1149         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1150                 vpnName, usedRds, vrfEntry.getDestPrefix());
1151         // create loadbalancing groups for extra routes only when the extra route is present behind
1152         // multiple VMs
1153         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1154                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1155             List<InstructionInfo> instructions = new ArrayList<>();
1156             // Obtain the local routes for this particular dpn.
1157             java.util.Optional<Routes> routes = vpnExtraRoutes
1158                     .stream()
1159                     .filter(route -> {
1160                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1161                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1162                         if (prefixToInterface == null) {
1163                             return false;
1164                         }
1165                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1166                     }).findFirst();
1167             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1168                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1169             if (groupId == FibConstants.INVALID_GROUP_ID) {
1170                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1171                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1172                 return;
1173             }
1174             List<ActionInfo> actionInfos =
1175                     Collections.singletonList(new ActionGroup(groupId));
1176             instructions.add(new InstructionApplyActions(actionInfos));
1177             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1178                     NwConstants.ADD_FLOW, tx, null);
1179         } else {
1180             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1181         }
1182
1183         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1184     }
1185
1186     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1187     /* Get interface info from prefix to interface mapping;
1188         Use the interface info to get the corresponding vpn interface op DS entry,
1189         remove the adjacency corresponding to this fib entry.
1190         If adjacency removed is the last adjacency, clean up the following:
1191          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1192          - prefix to interface entry
1193          - vpn interface op DS
1194      */
1195         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1196         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1197         if (prefixInfo == null) {
1198             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1199             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1200             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1201             if (extraRoute != null) {
1202                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1203                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1204                     if (nextHopIp != null) {
1205                         String ipPrefix;
1206                         if (isIpv4Address(nextHopIp)) {
1207                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1208                         } else {
1209                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1210                         }
1211                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1212                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1213                     }
1214                 }
1215             }
1216             if (prefixInfo == null) {
1217                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1218                 if (optionalLabel.isPresent()) {
1219                     Long label = optionalLabel.get();
1220                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1221                     LabelRouteInfo lri = getLabelRouteInfo(label);
1222                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1223                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1224                         prefixBuilder.setDpnId(lri.getDpnId());
1225                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1226                         prefixBuilder.setIpAddress(lri.getPrefix());
1227                         prefixInfo = prefixBuilder.build();
1228                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1229                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1230                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1231                     }
1232                 }
1233             }
1234         } else {
1235             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1236         }
1237     }
1238
1239     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1240                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1241
1242         if (prefixInfo == null) {
1243             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1244                     vrfEntry.getDestPrefix(), vpnId, rd);
1245             return; //Don't have any info for this prefix (shouldn't happen); need to return
1246         }
1247
1248         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1249             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1250                     vrfEntry.getDestPrefix(), vpnId, rd);
1251             return;
1252         }
1253
1254         String ifName = prefixInfo.getVpnInterfaceName();
1255         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1256             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1257     }
1258
1259     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1260         Prefixes prefixInfo;
1261         Long vpnId;
1262         String rd;
1263         VrfEntry vrfEntry;
1264         Routes extraRoute;
1265
1266         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1267                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1268             this.prefixInfo = prefixInfo;
1269             this.vpnId = vpnId;
1270             this.rd = rd;
1271             this.vrfEntry = vrfEntry;
1272             this.extraRoute = extraRoute;
1273         }
1274
1275         @Override
1276         public List<ListenableFuture<Void>> call() {
1277             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1278             // to call the respective helpers.
1279             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1280                 //First Cleanup LabelRouteInfo
1281                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1282                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1283                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1284                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1285                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1286                         synchronized (label.toString().intern()) {
1287                             LabelRouteInfo lri = getLabelRouteInfo(label);
1288                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1289                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1290                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1291                                         fibUtil.getVpnInstanceOpData(rd);
1292                                 String vpnInstanceName = "";
1293                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1294                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1295                                 }
1296                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1297                                 if (lriRemoved) {
1298                                     String parentRd = lri.getParentVpnRd();
1299                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1300                                             parentRd, vrfEntry.getDestPrefix()));
1301                                 }
1302                             } else {
1303                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1304                                         rd, vrfEntry.getDestPrefix()));
1305                             }
1306                         }
1307                     });
1308                 }
1309                 String ifName = prefixInfo.getVpnInterfaceName();
1310                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1311                 String vpnName = null;
1312
1313                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1314                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1315                     * */
1316                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1317                     if (vpnNameOpt.isPresent()) {
1318                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1319                     }
1320                 }
1321                 if (optVpnName.isPresent()) {
1322                     vpnName = optVpnName.get();
1323                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1324                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1325                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1326                     if (opVpnInterface.isPresent()) {
1327                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1328                         if (vpnId != associatedVpnId) {
1329                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1330                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1331                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1332                             return;
1333                         } else {
1334                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1335                                     vrfEntry.getDestPrefix(), associatedVpnId);
1336                         }
1337                     }
1338                 }
1339                 if (extraRoute != null) {
1340                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1341                     //Only one used Rd present in case of removal event
1342                     String usedRd = usedRds.get(0);
1343                     if (optVpnName.isPresent()) {
1344                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1345                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1346                                         vrfEntry.getDestPrefix()));
1347                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1348                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1349                     }
1350                 }
1351                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1352             }));
1353         }
1354     }
1355
1356     /**
1357      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1358      * Remove Adjacency from VPNInterfaceOpData.
1359      * if Adjacency != primary.
1360      * if Adjacency == primary , then mark it for deletion.
1361      * Remove entire VPNinterfaceOpData Entry.
1362      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1363      * @param vrfEntry - VrfEntry removed
1364      * @param ifName - Interface name from VRFentry
1365      * @param vpnName - VPN name of corresponding VRF
1366      * @param tx - ReadWrite Tx
1367      * @throws ReadFailedException - Exception thrown in case of read failed
1368      */
1369     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1370                                                           ReadWriteTransaction tx) throws ReadFailedException {
1371         InstanceIdentifier<Adjacency> adjacencyIid =
1372                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1373         Optional<Adjacency> adjacencyOptional = tx.read(LogicalDatastoreType.OPERATIONAL, adjacencyIid).checkedGet();
1374         if (adjacencyOptional.isPresent()) {
1375             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1376                 tx.delete(LogicalDatastoreType.OPERATIONAL,
1377                         FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1378             } else {
1379                 tx.merge(LogicalDatastoreType.OPERATIONAL, adjacencyIid,
1380                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1381             }
1382         }
1383
1384         Optional<AdjacenciesOp> optAdjacencies =
1385                 tx.read(LogicalDatastoreType.OPERATIONAL,
1386                         FibUtil.getAdjListPathOp(ifName, vpnName)).checkedGet();
1387
1388         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1389             return;
1390         }
1391
1392         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1393                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1394                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1395                         && adjacency.isMarkedForDeletion() != null
1396                         && adjacency.isMarkedForDeletion()
1397         )) {
1398             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1399             tx.delete(LogicalDatastoreType.OPERATIONAL,
1400                     FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1401         }
1402     }
1403
1404     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1405         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1406         final String rd = vrfTableKey.getRouteDistinguisher();
1407         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1408         if (vpnInstance == null) {
1409             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1410             return;
1411         }
1412         final Collection<VpnToDpnList> vpnToDpnList;
1413         if (vrfEntry.getParentVpnRd() != null
1414                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1415             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1416             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1417             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1418                     vpnInstance.getVpnToDpnList();
1419             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1420                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1421         } else {
1422             vpnToDpnList = vpnInstance.getVpnToDpnList();
1423         }
1424
1425         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1426         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1427         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1428         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1429         if (subnetRoute != null) {
1430             long elanTag = subnetRoute.getElantag();
1431             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1432                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1433             if (vpnToDpnList != null) {
1434                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1435                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1436                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1437
1438                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1439                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1440                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1441                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1442                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1443                             }
1444
1445                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1446                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1447                         }
1448                     })));
1449             }
1450             optionalLabel.ifPresent(label -> {
1451                 synchronized (label.toString().intern()) {
1452                     LabelRouteInfo lri = getLabelRouteInfo(label);
1453                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1454                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1455                                 fibUtil.getVpnInstanceOpData(rd);
1456                         String vpnInstanceName = "";
1457                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1458                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1459                         }
1460                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1461                         if (lriRemoved) {
1462                             String parentRd = lri.getParentVpnRd();
1463                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1464                                     parentRd, vrfEntry.getDestPrefix()));
1465                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1466                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1467                         }
1468                     } else {
1469                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1470                                 rd, vrfEntry.getDestPrefix()));
1471                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1472                                 label, rd, vrfEntry.getDestPrefix());
1473                     }
1474                 }
1475             });
1476             return;
1477         }
1478
1479         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1480             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1481         if (vpnToDpnList != null) {
1482             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1483                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1484             String jobKey;
1485             Optional<Routes> extraRouteOptional;
1486             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1487             if (usedRds != null && !usedRds.isEmpty()) {
1488                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1489                 if (usedRds.size() > 1) {
1490                     LOG.error("The extra route prefix is still present in some DPNs");
1491                     return ;
1492                 } else {
1493                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1494                     //is not present in any other DPN
1495                     extraRouteOptional = VpnExtraRouteHelper
1496                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1497                 }
1498             } else {
1499                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1500                 extraRouteOptional = Optional.absent();
1501             }
1502
1503             jobCoordinator.enqueueJob(jobKey,
1504                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1505                     if (localDpnIdList.size() <= 0) {
1506                         for (VpnToDpnList curDpn : vpnToDpnList) {
1507                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1508                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1509                         }
1510                     } else {
1511                         for (BigInteger localDpnId : localDpnIdList) {
1512                             for (VpnToDpnList curDpn : vpnToDpnList) {
1513                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1514                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1515                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1516                                 }
1517                             }
1518                         }
1519                     }
1520                 })), MAX_RETRIES);
1521         }
1522
1523         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1524         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1525         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1526
1527         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1528         // of the interVpnLink.
1529         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1530         if (optVpnUuid.isPresent()) {
1531             String vpnUuid = optVpnUuid.get();
1532             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1533                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1534                 if (optInterVpnLink.isPresent()) {
1535                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1536                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1537                         // This is route that points to the other endpoint of an InterVpnLink
1538                         // In that case, we should look for the FIB table pointing to
1539                         // LPortDispatcher table and remove it.
1540                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1541                     }
1542                 }
1543             });
1544         }
1545
1546     }
1547
1548     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1549                                     int addOrRemove, WriteTransaction tx) {
1550         if (tx == null) {
1551             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1552                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1553                 "Error making LFIB table entry");
1554             return;
1555         }
1556
1557         List<MatchInfo> matches = new ArrayList<>();
1558         matches.add(MatchEthernetType.MPLS_UNICAST);
1559         matches.add(new MatchMplsLabel(label));
1560
1561         // Install the flow entry in L3_LFIB_TABLE
1562         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1563
1564         FlowEntity flowEntity;
1565         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1566             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1567         Flow flow = flowEntity.getFlowBuilder().build();
1568         String flowId = flowEntity.getFlowId();
1569         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1570         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1571         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1572             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1573             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1574
1575         if (addOrRemove == NwConstants.ADD_FLOW) {
1576             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1577         } else {
1578             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1579         }
1580
1581         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1582             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1583     }
1584
1585     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1586                                     final FutureCallback<List<Void>> callback) {
1587         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1588         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1589             () -> {
1590                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1591                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1592                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1593                         LogicalDatastoreType.CONFIGURATION, id);
1594                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1595                 if (!vrfTable.isPresent()) {
1596                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1597                     if (callback != null) {
1598                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1599                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1600                     }
1601                     return futures;
1602                 }
1603                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1604                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1605                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1606                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1607                             if (subnetRoute != null) {
1608                                 long elanTag = subnetRoute.getElantag();
1609                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1610                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1611                                         tx);
1612                                 continue;
1613                             }
1614                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1615                             if (routerInt != null) {
1616                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1617                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1618                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1619                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1620                                         NwConstants.ADD_FLOW);
1621                                 continue;
1622                             }
1623                             //Handle local flow creation for imports
1624                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1625                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1626                                 if (optionalLabel.isPresent()) {
1627                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1628                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1629                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1630                                         if (lri.getDpnId().equals(dpnId)) {
1631                                             createLocalFibEntry(vpnId, rd, vrfEntry);
1632                                             continue;
1633                                         }
1634                                     }
1635                                 }
1636                             }
1637                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1638                                     vrfEntry, dpnId);
1639                             if (shouldCreateRemoteFibEntry) {
1640                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1641                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1642                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1643                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1644                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1645                                 } else {
1646                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1647                                             vrfEntry, tx);
1648                                 }
1649                             }
1650                         }
1651                     }));
1652                     if (callback != null) {
1653                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1654                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1655                     }
1656                 }
1657                 return futures;
1658             });
1659     }
1660
1661     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1662                                             final String localNextHopIp, final String remoteNextHopIp) {
1663         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1664             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1665         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1666         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1667         List<SubTransaction> txnObjects =  new ArrayList<>();
1668         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1669         if (vrfTable.isPresent()) {
1670             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1671                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1672                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1673                         vrfTable.get().getVrfEntry().stream()
1674                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1675                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1676                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1677                     }
1678                 })));
1679         }
1680     }
1681
1682     public void manageRemoteRouteOnDPN(final boolean action,
1683                                        final BigInteger localDpnId,
1684                                        final long vpnId,
1685                                        final String rd,
1686                                        final String destPrefix,
1687                                        final String destTepIp,
1688                                        final long label) {
1689         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1690
1691         if (vpnInstance == null) {
1692             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1693             return;
1694         }
1695
1696         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1697             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1698                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1699                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1700                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1701                     if (vrfEntry == null) {
1702                         return;
1703                     }
1704                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1705                             action, localDpnId, vpnId, rd, destPrefix);
1706                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1707                     VrfEntry modVrfEntry;
1708                     if (routePathList == null || routePathList.isEmpty()) {
1709                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1710                                 Collections.singletonList(destTepIp),
1711                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1712                     } else {
1713                         modVrfEntry = vrfEntry;
1714                     }
1715
1716                     if (action) {
1717                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1718                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1719                                 modVrfEntry, tx);
1720                     } else {
1721                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1722                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1723                                 vrfEntry.getDestPrefix());
1724                         if (usedRds.size() > 1) {
1725                             LOG.debug("The extra route prefix is still present in some DPNs");
1726                             return;
1727                         }
1728                         //Is this fib route an extra route? If yes, get the nexthop which would be
1729                         //an adjacency in the vpn
1730                         Optional<Routes> extraRouteOptional = Optional.absent();
1731                         if (usedRds.size() != 0) {
1732                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1733                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1734                                     usedRds.get(0), vrfEntry.getDestPrefix());
1735                         }
1736                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1737                                 extraRouteOptional, tx);
1738                     }
1739                 }
1740             })));
1741     }
1742
1743     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1744                                  final FutureCallback<List<Void>> callback) {
1745         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1746         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1747             () -> {
1748                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1749                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1750                 List<SubTransaction> txnObjects =  new ArrayList<>();
1751                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1752                         LogicalDatastoreType.CONFIGURATION, id);
1753                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1754                 if (vrfTable.isPresent()) {
1755                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1756                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1757                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1758                                 /* Handle subnet routes here */
1759                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1760                                 if (subnetRoute != null) {
1761                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1762                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1763                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1764                                             NwConstants.DEL_FLOW, tx, null);
1765                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1766                                     if (routePaths != null) {
1767                                         for (RoutePaths routePath : routePaths) {
1768                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1769                                                     DEFAULT_FIB_FLOW_PRIORITY,
1770                                                     NwConstants.DEL_FLOW, tx);
1771                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1772                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1773                                                     vrfEntry.getDestPrefix());
1774                                         }
1775                                     }
1776                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1777                                             NwConstants.DEL_FLOW, tx);
1778                                     continue;
1779                                 }
1780                                 // ping responder for router interfaces
1781                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1782                                 if (routerInt != null) {
1783                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1784                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1785                                             routerInt.getMacAddress());
1786                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1787                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1788                                             NwConstants.DEL_FLOW);
1789                                     continue;
1790                                 }
1791
1792                                 //Handle local flow deletion for imports
1793                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1794                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1795                                     if (optionalLabel.isPresent()) {
1796                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1797                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1798                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1799                                                 lri) && lri.getDpnId().equals(dpnId)) {
1800                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1801                                         }
1802                                     }
1803                                 }
1804
1805                                 // Passing null as we don't know the dpn
1806                                 // to which prefix is attached at this point
1807                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1808                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1809                                 String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1810                                 Optional<Routes> extraRouteOptional;
1811                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1812                                 //an adjacency in the vpn
1813                                 if (usedRds != null && !usedRds.isEmpty()) {
1814                                     if (usedRds.size() > 1) {
1815                                         LOG.error("The extra route prefix is still present in some DPNs");
1816                                         return;
1817                                     } else {
1818                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1819                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1820
1821                                     }
1822                                 } else {
1823                                     extraRouteOptional = Optional.absent();
1824                                 }
1825                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1826                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1827                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1828                                 } else {
1829                                     baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(),
1830                                             vrfEntry, extraRouteOptional, tx);
1831                                 }
1832                             }
1833                         }));
1834                     }
1835                     if (callback != null) {
1836                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1837                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1838                     }
1839                 } else {
1840                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1841                 }
1842                 return futures;
1843             });
1844
1845     }
1846
1847     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1848                                            final String localNextHopIp, final String remoteNextHopIp) {
1849         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1850                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1851             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1852         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1853         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1854         List<SubTransaction> txnObjects =  new ArrayList<>();
1855         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1856         if (vrfTable.isPresent()) {
1857             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1858                 () -> {
1859                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1860                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1861                             tx -> vrfTable.get().getVrfEntry().stream()
1862                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1863                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1864                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1865                     }
1866                 });
1867         }
1868     }
1869
1870     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1871         InstanceIdentifierBuilder<VrfTables> idBuilder =
1872             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1873         return idBuilder.build();
1874     }
1875
1876     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1877         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1878                 .FLOWID_SEPARATOR + nextHop;
1879     }
1880
1881     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1882         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1883                 + tableMiss + FLOWID_PREFIX;
1884     }
1885
1886     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1887         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1888             .child(VrfTables.class, new VrfTablesKey(rd))
1889             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1890         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1891         if (vrfEntry.isPresent()) {
1892             return vrfEntry.get();
1893         }
1894         return null;
1895     }
1896
1897     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1898                                              final String vpnName,
1899                                              final VrfEntry vrfEntry) {
1900         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1901
1902         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1903         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1904
1905         if (targetDpns.isEmpty()) {
1906             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1907             return;
1908         }
1909
1910         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1911         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1912
1913         // delete from FIB
1914         //
1915         optNextHop.ifPresent(nextHop -> {
1916             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1917             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1918             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1919                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1920
1921             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1922             for (BigInteger dpId : targetDpns) {
1923                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1924                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1925
1926                 mdsalManager.removeFlow(dpId, flow);
1927             }
1928         });
1929
1930         // delete from LFIB
1931         //
1932         optLabel.ifPresent(label -> {
1933             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1934
1935             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1936                 for (BigInteger dpId : targetDpns) {
1937                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1938                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1939                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1940                             tx);
1941                 }
1942             }), LOG, "Error removing flows");
1943         });
1944     }
1945
1946     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1947             List<String> nextHopAddressList, LabelRouteInfo lri) {
1948         return lri != null && lri.getPrefix().equals(prefix)
1949                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1950     }
1951
1952     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1953         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1954             return true;
1955         }
1956
1957         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1958         if (prefix != null) {
1959             BigInteger prefixDpnId = prefix.getDpnId();
1960             if (dpnId.equals(prefixDpnId)) {
1961                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1962                         vrfEntry, dpnId);
1963                 return false;
1964             }
1965         }
1966         return true;
1967     }
1968 }