fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  42 import org.opendaylight.genius.mdsalutil.ActionInfo;
  43 import org.opendaylight.genius.mdsalutil.FlowEntity;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  48 import org.opendaylight.genius.mdsalutil.NWUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.UpgradeState;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  54 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  57 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  63 import org.opendaylight.genius.utils.ServiceIndex;
  64 import org.opendaylight.genius.utils.batching.SubTransaction;
  65 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  66 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  67 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  68 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  69 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  70 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  71 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  72 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  73 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  74 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 110 import org.slf4j.Logger;
 111 import org.slf4j.LoggerFactory;
 112
 113
 114 @Singleton
 115 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 118     private static final String FLOWID_PREFIX = "L3.";
 119     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 120     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 121     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 122     private static final int LFIB_INTERVPN_PRIORITY = 15;
 123     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 124     private static final int MAX_RETRIES = 3;
 125     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 126
 127     private final DataBroker dataBroker;
 128     private final ManagedNewTransactionRunner txRunner;
 129     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 130     private final IMdsalApiManager mdsalManager;
 131     private final NexthopManager nextHopManager;
 132     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 133     private final BaseVrfEntryHandler baseVrfEntryHandler;
 134     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 135     private final JobCoordinator jobCoordinator;
 136     private final IElanService elanManager;
 137     private final FibUtil fibUtil;
 138     private final InterVpnLinkCache interVpnLinkCache;
 139     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 140     private final UpgradeState upgradeState;
 141     private final DataTreeEventCallbackRegistrar eventCallbacks;
 142
 143     @Inject
 144     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 145                             final NexthopManager nexthopManager,
 146                             final IElanService elanManager,
 147                             final BaseVrfEntryHandler vrfEntryHandler,
 148                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 149                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 150                             final JobCoordinator jobCoordinator,
 151                             final FibUtil fibUtil,
 152                             final InterVpnLinkCache interVpnLinkCache,
 153                             final UpgradeState upgradeState,
 154                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 155         super(VrfEntry.class, VrfEntryListener.class);
 156         this.dataBroker = dataBroker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 158         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 159         this.mdsalManager = mdsalApiManager;
 160         this.nextHopManager = nexthopManager;
 161         this.elanManager = elanManager;
 162         this.baseVrfEntryHandler = vrfEntryHandler;
 163         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 164         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.fibUtil = fibUtil;
 167         this.interVpnLinkCache = interVpnLinkCache;
 168         this.upgradeState = upgradeState;
 169         this.eventCallbacks = eventCallbacks;
 170     }
 171
 172     @Override
 173     @PostConstruct
 174     public void init() {
 175         LOG.info("{} init", getClass().getSimpleName());
 176         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 177     }
 178
 179     @Override
 180     @SuppressWarnings("checkstyle:IllegalCatch")
 181     public void close() {
 182         closeables.forEach(c -> {
 183             try {
 184                 c.close();
 185             } catch (Exception e) {
 186                 LOG.warn("Error closing {}", c, e);
 187             }
 188         });
 189     }
 190
 191     @Override
 192     protected VrfEntryListener getDataTreeChangeListener() {
 193         return VrfEntryListener.this;
 194     }
 195
 196     @Override
 197     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 198         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 199     }
 200
 201     @Override
 202     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 203         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 204         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 205         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 206                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 207         addFibEntries(identifier, vrfEntry, rd);
 208         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 209                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 210     }
 211
 212     //This method is temporary. Eventually Factory design pattern will be used to get
 213     // right VrfEntryhandle and invoke its methods.
 214     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 215         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 216             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 217             return;
 218         }
 219         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 220             LOG.info("EVPN flows need to be programmed.");
 221             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 222                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 223             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 224             closeables.add(evpnVrfEntryHandler);
 225             return;
 226         }
 227         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 228         if (routerInt != null) {
 229             // ping responder for router interfaces
 230             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 234             createFibEntries(identifier, vrfEntry);
 235             return;
 236         }
 237     }
 238
 239     @Override
 240     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 241         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 242         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 243         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 244                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 245         removeFibEntries(identifier, vrfEntry, rd);
 246         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 247             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 248     }
 249
 250     //This method is temporary. Eventually Factory design pattern will be used to get
 251     // right VrfEntryhandle and invoke its methods.
 252     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 253         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 254             LOG.info("EVPN flows to be deleted");
 255             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 256                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 257             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 258             closeables.add(evpnVrfEntryHandler);
 259             return;
 260         }
 261         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 262         if (routerInt != null) {
 263             // ping responder for router interfaces
 264             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 265             return;
 266         }
 267         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 268             deleteFibEntries(identifier, vrfEntry);
 269             return;
 270         }
 271         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 272             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 273             return;
 274         }
 275     }
 276
 277     @Override
 278     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 279     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 280     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 281     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 282         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 283         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 284         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 285                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 286         // Handle BGP Routes first
 287         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 288             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 289             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 290                     rd, update.getDestPrefix(), update.getRoutePaths());
 291             return;
 292         }
 293
 294         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 295             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 296             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 297             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 298
 299             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 300             //updated vrf entry route path is null. This can happen during tunnel events.
 301             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 302             List<String> usedRds = new ArrayList<>();
 303             if (optVpnInstance.isPresent()) {
 304                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 305                         update.getDestPrefix());
 306             }
 307             // If original VRF Entry had nexthop null , but update VRF Entry
 308             // has nexthop , route needs to be created on remote Dpns
 309             if (originalRoutePath == null || originalRoutePath.isEmpty()
 310                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 311                 // TODO(vivek): Though ugly, Not handling this code now, as each
 312                 // tep add event will invoke flow addition
 313                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 314                         + " This event is IGNORED here.", update.getDestPrefix());
 315                 return;
 316             }
 317
 318             // If original VRF Entry had valid nexthop , but update VRF Entry
 319             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 320             if (updateRoutePath == null || updateRoutePath.isEmpty()
 321                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 322                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 323                         + "This event is IGNORED here.", update.getDestPrefix());
 324                 return;
 325             }
 326             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 327             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 328             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 329             ListenableFuture<Void> future =
 330                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> nextHopsRemoved.parallelStream()
 331                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 332                                     tx, nextHopRemoved, rd, update.getDestPrefix())));
 333             Futures.addCallback(future, new FutureCallback<Void>() {
 334                 @Override
 335                 public void onSuccess(Void result) {
 336                     createFibEntries(identifier, update);
 337                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 338                             rd, update.getDestPrefix(), update.getRoutePaths());
 339                 }
 340
 341                 @Override
 342                 public void onFailure(Throwable throwable) {
 343                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 344                             update, throwable);
 345                 }
 346             }, MoreExecutors.directExecutor());
 347             return;
 348         }
 349
 350         //Handle all other routes only on a cluster reboot
 351         if (original.equals(update)) {
 352             //Reboot use-case
 353             createFibEntries(identifier, update);
 354             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 355                     rd, update.getDestPrefix(), update.getRoutePaths());
 356             return;
 357         }
 358
 359         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 360                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 361     }
 362
 363     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 364         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 365         List<SubTransaction> txnObjects =  new ArrayList<>();
 366         final VpnInstanceOpDataEntry vpnInstance =
 367                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 368         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 369         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 370                 + " has null vpnId!");
 371         final Collection<VpnToDpnList> vpnToDpnList;
 372         if (vrfEntry.getParentVpnRd() != null
 373                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 374             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 375             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 376             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 377                 vpnInstance.getVpnToDpnList();
 378             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 379                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 380         } else {
 381             vpnToDpnList = vpnInstance.getVpnToDpnList();
 382         }
 383         final Long vpnId = vpnInstance.getVpnId();
 384         final String rd = vrfTableKey.getRouteDistinguisher();
 385         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 386         if (subnetRoute != null) {
 387             final long elanTag = subnetRoute.getElantag();
 388             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 389                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 390             if (vpnToDpnList != null) {
 391                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 392                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 393                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 394                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 395                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 396                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 397                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 398                             }
 399                         }
 400                     })));
 401             }
 402             return;
 403         }
 404         // Get etherType value based on the IpPrefix address family type
 405         int etherType;
 406         try {
 407             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 408         } catch (IllegalArgumentException ex) {
 409             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 410             return;
 411         }
 412
 413         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 414         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 415             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 416                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 417                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 418                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 419                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 420                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 421                                     try {
 422                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 423                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 424                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 425                                                     txnObjects);
 426                                         } else {
 427                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 428                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 429                                         }
 430                                     } catch (NullPointerException e) {
 431                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 432                                                 vrfEntry.getDestPrefix(), e);
 433                                     }
 434                                 }
 435                             }
 436                         }
 437                     }
 438                 })), MAX_RETRIES);
 439         }
 440
 441         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 442         if (optVpnUuid.isPresent()) {
 443             String vpnUuid = optVpnUuid.get();
 444             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 445             if (interVpnLink != null) {
 446                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 447                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 448                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 449                         // This is an static route that points to the other endpoint of an InterVpnLink
 450                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 451                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 452                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 453                     }
 454                 });
 455             }
 456         }
 457     }
 458
 459     void refreshFibTables(String rd, String prefix) {
 460         InstanceIdentifier<VrfEntry> vrfEntryId =
 461                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 462                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 463         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 464         if (vrfEntry.isPresent()) {
 465             createFibEntries(vrfEntryId, vrfEntry.get());
 466         }
 467     }
 468
 469     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 470         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 471         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 472         prefixBuilder.setDpnId(lri.getDpnId());
 473         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 474         prefixBuilder.setIpAddress(lri.getPrefix());
 475         // Increment the refCount here
 476         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 477             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 478         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 479         if (!isPresentInList) {
 480             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 481             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 482             vpnInstanceNames.add(vpnInstanceName);
 483             builder.setVpnInstanceList(vpnInstanceNames);
 484             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 485         } else {
 486             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 487         }
 488         return prefixBuilder.build();
 489     }
 490
 491     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 492                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 493         if (tx == null) {
 494             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 495                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 496                 "Error installing subnet route in FIB");
 497             return;
 498         }
 499         int etherType;
 500         try {
 501             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 502         } catch (IllegalArgumentException ex) {
 503             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 504             return;
 505         }
 506         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 507             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 508             synchronized (label.toString().intern()) {
 509                 LabelRouteInfo lri = getLabelRouteInfo(label);
 510                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 511
 512                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 513                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 514                                 fibUtil.getVpnInstanceOpData(rd);
 515                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 516                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 517                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 518                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 519                             }
 520                         }
 521                     }
 522                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 523                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 524                 }
 525             }
 526         });
 527         final List<InstructionInfo> instructions = new ArrayList<>();
 528         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 529             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 530         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 531         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 532         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 533                 NwConstants.ADD_FLOW, tx, null);
 534         if (vrfEntry.getRoutePaths() != null) {
 535             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 536                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 537                     List<ActionInfo> actionsInfos = new ArrayList<>();
 538                     // reinitialize instructions list for LFIB Table
 539                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 540                     actionsInfos.add(new ActionPopMpls(etherType));
 541                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 542                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 543                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 544                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 545
 546                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 547                             NwConstants.ADD_FLOW, tx);
 548                 }
 549             }
 550         }
 551     }
 552
 553     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 554                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 555         List<MatchInfo> matches = new ArrayList<>();
 556
 557         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 558                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 559         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 560         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 561         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 562
 563         InetAddress destPrefix;
 564         try {
 565             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 566         } catch (UnknownHostException e) {
 567             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 568                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 569             return;
 570         }
 571
 572         // Match on VpnId and SubnetBroadCast IP address
 573         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 574         matches.add(MatchEthernetType.IPV4);
 575
 576         if (prefixLength != 0) {
 577             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 578         }
 579
 580         //Action is to drop the packet
 581         List<InstructionInfo> dropInstructions = new ArrayList<>();
 582         List<ActionInfo> actionsInfos = new ArrayList<>();
 583         actionsInfos.add(new ActionDrop());
 584         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 585
 586         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 587         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 588         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 589                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 590
 591         Flow flow = flowEntity.getFlowBuilder().build();
 592         String flowId = flowEntity.getFlowId();
 593         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 594         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 595
 596         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 597                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 598                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 599
 600         if (addOrRemove == NwConstants.ADD_FLOW) {
 601             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 602         } else {
 603             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 604         }
 605     }
 606
 607     /*
 608      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 609      * LportDispatcher table (via table 80)
 610      */
 611     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 612                                             final VrfEntry vrfEntry, int etherType) {
 613         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 614         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 615         // packet is commuted from Vpn2 to Vpn1.
 616         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 617         if (!interVpnLink.isActive()) {
 618             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 619                      interVpnLinkName, vrfEntry.getDestPrefix());
 620             return;
 621         }
 622
 623         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 624         if (!optLportTag.isPresent()) {
 625             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 626             return;
 627         }
 628
 629         Long lportTag = optLportTag.get();
 630         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 631         if (label == null) {
 632             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 633                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 634             return;
 635         }
 636         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 637         List<InstructionInfo> instructions = Arrays.asList(
 638             new InstructionApplyActions(actionsInfos),
 639             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 640                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 641                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 642                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 643             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 644         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 645         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 646
 647         for (BigInteger dpId : targetDpns) {
 648             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 649                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 650
 651             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 652                                /*writeTx*/null);
 653         }
 654     }
 655
 656
 657     /*
 658      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 659      */
 660     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 661                                                final VrfEntry vrfEntry, long vpnTag) {
 662         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 663         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 664             && vrfEntry.getRoutePaths().size() == 1);
 665         String destination = vrfEntry.getDestPrefix();
 666         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 667         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 668
 669         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 670         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 671         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 672         if (interVpnLink.getState().or(State.Error) != State.Active) {
 673             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 674                 destination, nextHop, interVpnLinkName);
 675             return;
 676         }
 677
 678         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 679         if (!optOtherEndpointLportTag.isPresent()) {
 680             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 681                 vpnUuid, interVpnLinkName);
 682             return;
 683         }
 684
 685         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 686         if (targetDpns.isEmpty()) {
 687             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 688                 vpnUuid, interVpnLinkName);
 689             return;
 690         }
 691
 692         String[] values = destination.split("/");
 693         String destPrefixIpAddress = values[0];
 694         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 695
 696         List<MatchInfo> matches = new ArrayList<>();
 697         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 698         matches.add(MatchEthernetType.IPV4);
 699
 700         if (prefixLength != 0) {
 701             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 702         }
 703
 704         List<Instruction> instructions =
 705             Arrays.asList(new InstructionWriteMetadata(
 706                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 707                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 708                             .L3VPN_SERVICE_INDEX)),
 709                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 710                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 711
 712         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 713         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 714         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 715             COOKIE_VM_FIB_TABLE, matches, instructions);
 716
 717         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 718             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 719
 720         for (BigInteger dpId : targetDpns) {
 721
 722             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 723                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 724                 dpId, interVpnLink.getInterVpnLinkName());
 725
 726             mdsalManager.installFlow(dpId, flowEntity);
 727         }
 728     }
 729
 730     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 731         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 732         String localNextHopIP = vrfEntry.getDestPrefix();
 733         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 734         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 735         if (localNextHopInfo == null) {
 736             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 737             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 738                     vpnName, usedRds, localNextHopIP);
 739             if (LOG.isDebugEnabled()) {
 740                 LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 741                         vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 742             }
 743             boolean localNextHopSeen = false;
 744             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 745             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 746                 String ipPrefix;
 747                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 748                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 749                 } else {
 750                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 751                 }
 752                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 753                     ipPrefix);
 754                 if (localNextHopInfoLocal != null) {
 755                     localNextHopSeen = true;
 756                     BigInteger dpnId =
 757                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 758                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 759                     returnLocalDpnId.add(dpnId);
 760                 }
 761             }
 762             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 763                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 764                 if (optionalLabel.isPresent()) {
 765                     Long label = optionalLabel.get();
 766                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 767                     synchronized (label.toString().intern()) {
 768                         LabelRouteInfo lri = getLabelRouteInfo(label);
 769                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 770                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 771                                     fibUtil.getVpnInstanceOpData(rd);
 772                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 773                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 774                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 775                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 776                                     localNextHopIP = lri.getPrefix();
 777                                 } else {
 778                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 779                                     localNextHopIP = lri.getPrefix();
 780                                 }
 781                             }
 782                             if (localNextHopInfo != null) {
 783                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 784                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 785                                 if (vpnExtraRoutes.isEmpty()) {
 786                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 787                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 788                                     returnLocalDpnId.add(dpnId);
 789                                 } else {
 790                                     for (Routes extraRoutes : vpnExtraRoutes) {
 791                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 792                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 793                                         returnLocalDpnId.add(dpnId);
 794                                     }
 795                                 }
 796                             }
 797                         }
 798                     }
 799                 }
 800             }
 801             if (returnLocalDpnId.isEmpty()) {
 802                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 803             }
 804         } else {
 805             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 806                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 807             if (dpnId != null) {
 808                 returnLocalDpnId.add(dpnId);
 809             }
 810         }
 811         return returnLocalDpnId;
 812     }
 813
 814     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 815                                                 final Long vpnId, final String rd,
 816                                                 final VrfEntry vrfEntry,
 817                                                 Routes routes, List<Routes> vpnExtraRoutes,
 818                                                 int etherType) {
 819         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 820         if (localNextHopInfo != null) {
 821             long groupId;
 822             long localGroupId;
 823             final BigInteger dpnId = localNextHopInfo.getDpnId();
 824             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 825                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 826                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 827                 return dpnId;
 828             }
 829             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 830                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 831                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 832                 return dpnId;
 833             }
 834             if (!isVpnPresentInDpn(rd, dpnId)) {
 835                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 836                         vpnId, rd, dpnId.toString());
 837                 return BigInteger.ZERO;
 838             }
 839             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 840             String prefix = vrfEntry.getDestPrefix();
 841             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 842             //The loadbalancing group is created only if the extra route has multiple nexthops
 843             //to avoid loadbalancing the discovered routes
 844             if (vpnExtraRoutes != null && routes != null) {
 845                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 846                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 847                 } else {
 848                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 849                 }
 850                 if (vpnExtraRoutes.size() > 1) {
 851                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 852                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 853                 } else if (routes.getNexthopIpList().size() > 1) {
 854                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 855                     localGroupId = groupId;
 856                 } else {
 857                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 858                             prefix, gwMacAddress);
 859                     localGroupId = groupId;
 860                 }
 861             } else {
 862                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 863                         gwMacAddress);
 864                 localGroupId = groupId;
 865             }
 866             if (groupId == FibConstants.INVALID_GROUP_ID) {
 867                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 868                         prefix, rd, interfaceName, dpnId.toString());
 869                 return BigInteger.ZERO;
 870             }
 871             final List<InstructionInfo> instructions = Collections.singletonList(
 872                     new InstructionApplyActions(
 873                             Collections.singletonList(new ActionGroup(groupId))));
 874             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 875                     new InstructionApplyActions(
 876                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(groupId))));
 877             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 878             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 879             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 880             jobCoordinator.enqueueJob(jobKey, () -> {
 881                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 882                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 883                             NwConstants.ADD_FLOW, tx, null);
 884                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 885                             vpnName, rd)) {
 886                         optLabel.ifPresent(label -> {
 887                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 888                                 LOG.debug(
 889                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 890                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 891                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 892                                         nextHopAddressList);
 893                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 894                                         NwConstants.ADD_FLOW, tx);
 895                                 // If the extra-route is reachable from VMs attached to the same switch,
 896                                 // then the tunnel table can point to the load balancing group.
 897                                 // If it is reachable from VMs attached to different switches,
 898                                 // then it should be pointing to one of the local group in order to avoid looping.
 899                                 if (vrfEntry.getRoutePaths().size() == 1) {
 900                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 901                                 } else {
 902                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 903                                 }
 904                             } else {
 905                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 906                                                 + "route. LFib and Terminating table entries will not be created.",
 907                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 908                             }
 909                         });
 910                     }
 911                 }));
 912             });
 913             return dpnId;
 914         }
 915         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 916                 vpnName);
 917         return BigInteger.ZERO;
 918     }
 919
 920     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 921         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 922         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 923         if (dpnInVpn.isPresent()) {
 924             return true;
 925         }
 926         return false;
 927     }
 928
 929     private LabelRouteInfo getLabelRouteInfo(Long label) {
 930         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 931             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 932         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 933         if (opResult.isPresent()) {
 934             return opResult.get();
 935         }
 936         return null;
 937     }
 938
 939     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 940         if (lri == null) {
 941             return true;
 942         }
 943
 944         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 945         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 946             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 947
 948         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 949             ? lri.getVpnInstanceList() : new ArrayList<>();
 950         if (vpnInstancesList.contains(vpnInstanceName)) {
 951             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 952             vpnInstancesList.remove(vpnInstanceName);
 953         }
 954         if (vpnInstancesList.isEmpty()) {
 955             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 956             if (tx != null) {
 957                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 958             } else {
 959                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 960             }
 961             return true;
 962         } else {
 963             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 964             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 965             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 966         }
 967         return false;
 968     }
 969
 970     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 971                                       WriteTransaction tx) {
 972         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 973
 974         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 975
 976         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 977             dpId, label, groupId);
 978     }
 979
 980     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 981                                                 WriteTransaction tx) {
 982         List<MatchInfo> mkMatches = new ArrayList<>();
 983
 984         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 985             destDpId, label, actionsInfos);
 986
 987         // Matching metadata
 988         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 989         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 990
 991         List<InstructionInfo> mkInstructions = new ArrayList<>();
 992         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 993
 994         FlowEntity terminatingServiceTableFlowEntity =
 995             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 996             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 997                 String.format("%s:%d", "TST Flow Entry ", label),
 998             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 999
1000         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1001
1002         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1003
1004         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1005         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1006             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1007             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1008             .child(Flow.class, flowKey).build();
1009         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
1010                 WriteTransaction.CREATE_MISSING_PARENTS);
1011     }
1012
1013     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1014         FlowEntity flowEntity;
1015         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1016         List<MatchInfo> mkMatches = new ArrayList<>();
1017         // Matching metadata
1018         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1019         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1020             NwConstants.INTERNAL_TUNNEL_TABLE,
1021             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1022             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1023             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1024         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1025         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1026         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1027             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1028             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1029
1030         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1031         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1032     }
1033
1034     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1035         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1036         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1037         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1038         boolean shouldUpdateNonEcmpLocalNextHop = true;
1039         if (localNextHopInfo == null) {
1040             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1041             if (usedRds.size() > 1) {
1042                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1043                         vrfEntry.getDestPrefix(), vpnName, rd);
1044                 return returnLocalDpnId;
1045             }
1046             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1047             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1048             //in the vpn
1049             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1050                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1051             if (extraRouteOptional.isPresent()) {
1052                 Routes extraRoute = extraRouteOptional.get();
1053                 String ipPrefix;
1054                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1055                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1056                 } else {
1057                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1058                 }
1059                 if (extraRoute.getNexthopIpList().size() > 1) {
1060                     shouldUpdateNonEcmpLocalNextHop = false;
1061                 }
1062                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1063                 if (localNextHopInfo != null) {
1064                     String localNextHopIP = localNextHopInfo.getIpAddress();
1065                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1066                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1067                     if (!dpnId.equals(BigInteger.ZERO)) {
1068                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1069                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1070                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1071                                 /*remove*/ false);
1072                         returnLocalDpnId.add(dpnId);
1073                     }
1074                 } else {
1075                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1076                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1077                 }
1078             }
1079
1080             if (localNextHopInfo == null) {
1081                 /* Imported VRF entry */
1082                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1083                 if (optionalLabel.isPresent()) {
1084                     Long label = optionalLabel.get();
1085                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1086                     LabelRouteInfo lri = getLabelRouteInfo(label);
1087                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1088                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1089                         prefixBuilder.setDpnId(lri.getDpnId());
1090                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1091                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1092                         if (!dpnId.equals(BigInteger.ZERO)) {
1093                             returnLocalDpnId.add(dpnId);
1094                         }
1095                     }
1096                 }
1097             }
1098
1099         } else {
1100             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1101             String localNextHopIP = localNextHopInfo.getIpAddress();
1102             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1103                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1104             if (!dpnId.equals(BigInteger.ZERO)) {
1105                 returnLocalDpnId.add(dpnId);
1106             }
1107         }
1108
1109         return returnLocalDpnId;
1110     }
1111
1112     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1113                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1114                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1115         if (localNextHopInfo != null) {
1116             final BigInteger dpnId = localNextHopInfo.getDpnId();
1117             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1118                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1119                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1120                 return dpnId;
1121             }
1122             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1123                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1124                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1125                 return dpnId;
1126             }
1127
1128             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1129                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1130                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1131                             NwConstants.DEL_FLOW, tx, null);
1132                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1133                             vpnId, rd)) {
1134                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1135                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1136                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1137                                         NwConstants.DEL_FLOW, tx);
1138                                 removeTunnelTableEntry(dpnId, label, tx);
1139                             });
1140                         }
1141                     }
1142                 })));
1143             //TODO: verify below adjacency call need to be optimized (?)
1144             //In case of the removal of the extra route, the loadbalancing group is updated
1145             if (shouldUpdateNonEcmpLocalNextHop) {
1146                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1147             }
1148             return dpnId;
1149         }
1150         return BigInteger.ZERO;
1151     }
1152
1153     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1154             final VrfEntry vrfEntry, WriteTransaction tx) {
1155         if (tx == null) {
1156             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1157                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1158             }), LOG, "Error creating remote FIB entry");
1159             return;
1160         }
1161
1162         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1163         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1164                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1165
1166         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1167         if (adjacencyResults.isEmpty()) {
1168             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1169                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1170             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1171             return;
1172         }
1173
1174         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1175         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1176                 vpnName, usedRds, vrfEntry.getDestPrefix());
1177         // create loadbalancing groups for extra routes only when the extra route is present behind
1178         // multiple VMs
1179         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1180                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1181             List<InstructionInfo> instructions = new ArrayList<>();
1182             // Obtain the local routes for this particular dpn.
1183             java.util.Optional<Routes> routes = vpnExtraRoutes
1184                     .stream()
1185                     .filter(route -> {
1186                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1187                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1188                         if (prefixToInterface == null) {
1189                             return false;
1190                         }
1191                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1192                     }).findFirst();
1193             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1194                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1195             if (groupId == FibConstants.INVALID_GROUP_ID) {
1196                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1197                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1198                 return;
1199             }
1200             List<ActionInfo> actionInfos =
1201                     Collections.singletonList(new ActionGroup(groupId));
1202             instructions.add(new InstructionApplyActions(actionInfos));
1203             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1204                     NwConstants.ADD_FLOW, tx, null);
1205         } else {
1206             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1207         }
1208
1209         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1210     }
1211
1212     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1213     /* Get interface info from prefix to interface mapping;
1214         Use the interface info to get the corresponding vpn interface op DS entry,
1215         remove the adjacency corresponding to this fib entry.
1216         If adjacency removed is the last adjacency, clean up the following:
1217          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1218          - prefix to interface entry
1219          - vpn interface op DS
1220      */
1221         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1222         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1223         if (prefixInfo == null) {
1224             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1225             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1226             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1227             if (extraRoute != null) {
1228                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1229                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1230                     if (nextHopIp != null) {
1231                         String ipPrefix;
1232                         if (isIpv4Address(nextHopIp)) {
1233                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1234                         } else {
1235                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1236                         }
1237                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1238                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1239                     }
1240                 }
1241             }
1242             if (prefixInfo == null) {
1243                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1244                 if (optionalLabel.isPresent()) {
1245                     Long label = optionalLabel.get();
1246                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1247                     LabelRouteInfo lri = getLabelRouteInfo(label);
1248                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1249                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1250                         prefixBuilder.setDpnId(lri.getDpnId());
1251                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1252                         prefixBuilder.setIpAddress(lri.getPrefix());
1253                         prefixInfo = prefixBuilder.build();
1254                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1255                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1256                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1257                     }
1258                 }
1259             }
1260         } else {
1261             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1262         }
1263     }
1264
1265     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1266                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1267
1268         if (prefixInfo == null) {
1269             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1270                     vrfEntry.getDestPrefix(), vpnId, rd);
1271             return; //Don't have any info for this prefix (shouldn't happen); need to return
1272         }
1273
1274         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1275             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1276                     vrfEntry.getDestPrefix(), vpnId, rd);
1277             return;
1278         }
1279
1280         String ifName = prefixInfo.getVpnInterfaceName();
1281         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1282             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1283     }
1284
1285     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1286         Prefixes prefixInfo;
1287         Long vpnId;
1288         String rd;
1289         VrfEntry vrfEntry;
1290         Routes extraRoute;
1291
1292         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1293                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1294             this.prefixInfo = prefixInfo;
1295             this.vpnId = vpnId;
1296             this.rd = rd;
1297             this.vrfEntry = vrfEntry;
1298             this.extraRoute = extraRoute;
1299         }
1300
1301         @Override
1302         public List<ListenableFuture<Void>> call() {
1303             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1304             // to call the respective helpers.
1305             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1306                 //First Cleanup LabelRouteInfo
1307                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1308                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1309                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1310                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1311                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1312                         synchronized (label.toString().intern()) {
1313                             LabelRouteInfo lri = getLabelRouteInfo(label);
1314                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1315                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1316                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1317                                         fibUtil.getVpnInstanceOpData(rd);
1318                                 String vpnInstanceName = "";
1319                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1320                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1321                                 }
1322                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1323                                 if (lriRemoved) {
1324                                     String parentRd = lri.getParentVpnRd();
1325                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1326                                             parentRd, vrfEntry.getDestPrefix()));
1327                                 }
1328                             } else {
1329                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1330                                         rd, vrfEntry.getDestPrefix()));
1331                             }
1332                         }
1333                     });
1334                 }
1335                 String ifName = prefixInfo.getVpnInterfaceName();
1336                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1337                 String vpnName = null;
1338
1339                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1340                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1341                     * */
1342                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1343                     if (vpnNameOpt.isPresent()) {
1344                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1345                     }
1346                 }
1347                 if (optVpnName.isPresent()) {
1348                     vpnName = optVpnName.get();
1349                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1350                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1351                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1352                     if (opVpnInterface.isPresent()) {
1353                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1354                         if (vpnId != associatedVpnId) {
1355                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1356                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1357                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1358                             return;
1359                         } else {
1360                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1361                                     vrfEntry.getDestPrefix(), associatedVpnId);
1362                         }
1363                     }
1364                 }
1365                 if (extraRoute != null) {
1366                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1367                     //Only one used Rd present in case of removal event
1368                     String usedRd = usedRds.get(0);
1369                     if (optVpnName.isPresent()) {
1370                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1371                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1372                                         vrfEntry.getDestPrefix()));
1373                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1374                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1375                     }
1376                 }
1377                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1378             }));
1379         }
1380     }
1381
1382     /**
1383      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1384      * Remove Adjacency from VPNInterfaceOpData.
1385      * if Adjacency != primary.
1386      * if Adjacency == primary , then mark it for deletion.
1387      * Remove entire VPNinterfaceOpData Entry.
1388      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1389      * @param vrfEntry - VrfEntry removed
1390      * @param ifName - Interface name from VRFentry
1391      * @param vpnName - VPN name of corresponding VRF
1392      * @param tx - ReadWrite Tx
1393      * @throws ReadFailedException - Exception thrown in case of read failed
1394      */
1395     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1396                                                           ReadWriteTransaction tx) throws ReadFailedException {
1397         InstanceIdentifier<Adjacency> adjacencyIid =
1398                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1399         Optional<Adjacency> adjacencyOptional = tx.read(LogicalDatastoreType.OPERATIONAL, adjacencyIid).checkedGet();
1400         if (adjacencyOptional.isPresent()) {
1401             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1402                 tx.delete(LogicalDatastoreType.OPERATIONAL,
1403                         FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1404             } else {
1405                 tx.merge(LogicalDatastoreType.OPERATIONAL, adjacencyIid,
1406                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1407             }
1408         }
1409
1410         Optional<AdjacenciesOp> optAdjacencies =
1411                 tx.read(LogicalDatastoreType.OPERATIONAL,
1412                         FibUtil.getAdjListPathOp(ifName, vpnName)).checkedGet();
1413
1414         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1415             return;
1416         }
1417
1418         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1419                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1420                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1421                         && adjacency.isMarkedForDeletion() != null
1422                         && adjacency.isMarkedForDeletion()
1423         )) {
1424             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1425             tx.delete(LogicalDatastoreType.OPERATIONAL,
1426                     FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1427         }
1428     }
1429
1430     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1431         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1432         final String rd = vrfTableKey.getRouteDistinguisher();
1433         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1434         if (vpnInstance == null) {
1435             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1436             return;
1437         }
1438         final Collection<VpnToDpnList> vpnToDpnList;
1439         if (vrfEntry.getParentVpnRd() != null
1440                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1441             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1442             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1443             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1444                     vpnInstance.getVpnToDpnList();
1445             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1446                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1447         } else {
1448             vpnToDpnList = vpnInstance.getVpnToDpnList();
1449         }
1450
1451         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1452         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1453         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1454         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1455         if (subnetRoute != null) {
1456             long elanTag = subnetRoute.getElantag();
1457             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1458                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1459             if (vpnToDpnList != null) {
1460                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1461                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1462                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1463
1464                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1465                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1466                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1467                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1468                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1469                             }
1470
1471                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1472                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1473                         }
1474                     })));
1475             }
1476             optionalLabel.ifPresent(label -> {
1477                 synchronized (label.toString().intern()) {
1478                     LabelRouteInfo lri = getLabelRouteInfo(label);
1479                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1480                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1481                                 fibUtil.getVpnInstanceOpData(rd);
1482                         String vpnInstanceName = "";
1483                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1484                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1485                         }
1486                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1487                         if (lriRemoved) {
1488                             String parentRd = lri.getParentVpnRd();
1489                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1490                                     parentRd, vrfEntry.getDestPrefix()));
1491                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1492                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1493                         }
1494                     } else {
1495                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1496                                 rd, vrfEntry.getDestPrefix()));
1497                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1498                                 label, rd, vrfEntry.getDestPrefix());
1499                     }
1500                 }
1501             });
1502             return;
1503         }
1504
1505         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1506             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1507         if (vpnToDpnList != null) {
1508             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1509                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1510             String jobKey;
1511             Optional<Routes> extraRouteOptional;
1512             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1513             if (usedRds != null && !usedRds.isEmpty()) {
1514                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1515                 if (usedRds.size() > 1) {
1516                     LOG.error("The extra route prefix is still present in some DPNs");
1517                     return ;
1518                 } else {
1519                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1520                     //is not present in any other DPN
1521                     extraRouteOptional = VpnExtraRouteHelper
1522                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1523                 }
1524             } else {
1525                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1526                 extraRouteOptional = Optional.absent();
1527             }
1528
1529             jobCoordinator.enqueueJob(jobKey,
1530                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1531                     if (localDpnIdList.size() <= 0) {
1532                         for (VpnToDpnList curDpn : vpnToDpnList) {
1533                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1534                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1535                         }
1536                     } else {
1537                         for (BigInteger localDpnId : localDpnIdList) {
1538                             for (VpnToDpnList curDpn : vpnToDpnList) {
1539                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1540                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1541                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1542                                 }
1543                             }
1544                         }
1545                     }
1546                 })), MAX_RETRIES);
1547         }
1548
1549         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1550         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1551         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1552
1553         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1554         // of the interVpnLink.
1555         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1556         if (optVpnUuid.isPresent()) {
1557             String vpnUuid = optVpnUuid.get();
1558             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1559                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1560                 if (optInterVpnLink.isPresent()) {
1561                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1562                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1563                         // This is route that points to the other endpoint of an InterVpnLink
1564                         // In that case, we should look for the FIB table pointing to
1565                         // LPortDispatcher table and remove it.
1566                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1567                     }
1568                 }
1569             });
1570         }
1571
1572     }
1573
1574     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1575                                     int addOrRemove, WriteTransaction tx) {
1576         if (tx == null) {
1577             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1578                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1579                 "Error making LFIB table entry");
1580             return;
1581         }
1582
1583         List<MatchInfo> matches = new ArrayList<>();
1584         matches.add(MatchEthernetType.MPLS_UNICAST);
1585         matches.add(new MatchMplsLabel(label));
1586
1587         // Install the flow entry in L3_LFIB_TABLE
1588         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1589
1590         FlowEntity flowEntity;
1591         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1592             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1593         Flow flow = flowEntity.getFlowBuilder().build();
1594         String flowId = flowEntity.getFlowId();
1595         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1596         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1597         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1598             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1599             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1600
1601         if (addOrRemove == NwConstants.ADD_FLOW) {
1602             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1603         } else {
1604             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1605         }
1606
1607         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1608             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1609     }
1610
1611     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1612                                     final FutureCallback<List<Void>> callback) {
1613         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1614         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1615             () -> {
1616                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1617                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1618                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1619                         LogicalDatastoreType.CONFIGURATION, id);
1620                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1621                 if (!vrfTable.isPresent()) {
1622                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1623                     if (callback != null) {
1624                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1625                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1626                     }
1627                     return futures;
1628                 }
1629                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1630                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1631                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1632                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1633                             if (subnetRoute != null) {
1634                                 long elanTag = subnetRoute.getElantag();
1635                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1636                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1637                                         tx);
1638                                 continue;
1639                             }
1640                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1641                             if (routerInt != null) {
1642                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1643                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1644                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1645                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1646                                         NwConstants.ADD_FLOW);
1647                                 continue;
1648                             }
1649                             //Handle local flow creation for imports
1650                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1651                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1652                                 if (optionalLabel.isPresent()) {
1653                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1654                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1655                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1656                                         if (lri.getDpnId().equals(dpnId)) {
1657                                             try {
1658                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1659                                                         vrfEntry.getDestPrefix());
1660                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1661                                             } catch (IllegalArgumentException ex) {
1662                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1663                                                         vrfEntry.getDestPrefix());
1664                                             }
1665                                             continue;
1666                                         }
1667                                     }
1668                                 }
1669                             }
1670                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1671                                     vrfEntry, dpnId);
1672                             if (shouldCreateRemoteFibEntry) {
1673                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1674                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1675                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1676                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1677                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1678                                 } else {
1679                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1680                                             vrfEntry, tx);
1681                                 }
1682                             }
1683                         }
1684                     }));
1685                     if (callback != null) {
1686                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1687                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1688                     }
1689                 }
1690                 return futures;
1691             });
1692     }
1693
1694     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1695                                             final String localNextHopIp, final String remoteNextHopIp) {
1696         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1697             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1698         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1699         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1700         List<SubTransaction> txnObjects =  new ArrayList<>();
1701         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1702         if (vrfTable.isPresent()) {
1703             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1704                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1705                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1706                         vrfTable.get().getVrfEntry().stream()
1707                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1708                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1709                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1710                     }
1711                 })));
1712         }
1713     }
1714
1715     public void manageRemoteRouteOnDPN(final boolean action,
1716                                        final BigInteger localDpnId,
1717                                        final long vpnId,
1718                                        final String rd,
1719                                        final String destPrefix,
1720                                        final String destTepIp,
1721                                        final long label) {
1722         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1723
1724         if (vpnInstance == null) {
1725             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1726             return;
1727         }
1728
1729         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1730             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1731                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1732                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1733                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1734                     if (vrfEntry == null) {
1735                         return;
1736                     }
1737                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1738                             action, localDpnId, vpnId, rd, destPrefix);
1739                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1740                     VrfEntry modVrfEntry;
1741                     if (routePathList == null || routePathList.isEmpty()) {
1742                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1743                                 Collections.singletonList(destTepIp),
1744                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1745                     } else {
1746                         modVrfEntry = vrfEntry;
1747                     }
1748
1749                     if (action) {
1750                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1751                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1752                                 modVrfEntry, tx);
1753                     } else {
1754                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1755                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1756                                 vrfEntry.getDestPrefix());
1757                         if (usedRds.size() > 1) {
1758                             LOG.debug("The extra route prefix is still present in some DPNs");
1759                             return;
1760                         }
1761                         //Is this fib route an extra route? If yes, get the nexthop which would be
1762                         //an adjacency in the vpn
1763                         Optional<Routes> extraRouteOptional = Optional.absent();
1764                         if (usedRds.size() != 0) {
1765                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1766                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1767                                     usedRds.get(0), vrfEntry.getDestPrefix());
1768                         }
1769                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1770                                 extraRouteOptional, tx);
1771                     }
1772                 }
1773             })));
1774     }
1775
1776     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1777                                  final FutureCallback<List<Void>> callback) {
1778         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1779         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1780             () -> {
1781                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1782                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1783                 List<SubTransaction> txnObjects =  new ArrayList<>();
1784                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1785                         LogicalDatastoreType.CONFIGURATION, id);
1786                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1787                 if (vrfTable.isPresent()) {
1788                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1789                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1790                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1791                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1792                                 /* Handle subnet routes here */
1793                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1794                                 if (subnetRoute != null && !fibUtil
1795                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1796                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1797                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1798                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1799                                             NwConstants.DEL_FLOW, tx, null);
1800                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1801                                     if (routePaths != null) {
1802                                         for (RoutePaths routePath : routePaths) {
1803                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1804                                                     DEFAULT_FIB_FLOW_PRIORITY,
1805                                                     NwConstants.DEL_FLOW, tx);
1806                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1807                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1808                                                     vrfEntry.getDestPrefix());
1809                                         }
1810                                     }
1811                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1812                                             NwConstants.DEL_FLOW, tx);
1813                                     continue;
1814                                 }
1815                                 // ping responder for router interfaces
1816                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1817                                 if (routerInt != null) {
1818                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1819                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1820                                             routerInt.getMacAddress());
1821                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1822                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1823                                             NwConstants.DEL_FLOW);
1824                                     continue;
1825                                 }
1826
1827                                 //Handle local flow deletion for imports
1828                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1829                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1830                                     if (optionalLabel.isPresent()) {
1831                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1832                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1833                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1834                                                 lri) && lri.getDpnId().equals(dpnId)) {
1835                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1836                                         }
1837                                     }
1838                                 }
1839
1840                                 // Passing null as we don't know the dpn
1841                                 // to which prefix is attached at this point
1842                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1843                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1844                                 Optional<Routes> extraRouteOptional;
1845                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1846                                 //an adjacency in the vpn
1847                                 if (usedRds != null && !usedRds.isEmpty()) {
1848                                     if (usedRds.size() > 1) {
1849                                         LOG.error("The extra route prefix is still present in some DPNs");
1850                                         return;
1851                                     } else {
1852                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1853                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1854
1855                                     }
1856                                 } else {
1857                                     extraRouteOptional = Optional.absent();
1858                                 }
1859                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1860                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1861                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1862                                 } else {
1863                                     if (subnetRoute == null || !fibUtil
1864                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1865                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1866                                                 vrfTable.get().key(), vrfEntry, extraRouteOptional, tx);
1867                                     }
1868                                 }
1869                             }
1870                         }));
1871                     }
1872                     if (callback != null) {
1873                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1874                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1875                     }
1876                 } else {
1877                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1878                 }
1879                 return futures;
1880             });
1881
1882     }
1883
1884     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1885                                            final String localNextHopIp, final String remoteNextHopIp) {
1886         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1887                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1888             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1889         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1890         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1891         List<SubTransaction> txnObjects =  new ArrayList<>();
1892         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1893         if (vrfTable.isPresent()) {
1894             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1895                 () -> {
1896                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1897                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1898                             tx -> vrfTable.get().getVrfEntry().stream()
1899                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1900                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1901                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1902                     }
1903                 });
1904         }
1905     }
1906
1907     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1908         InstanceIdentifierBuilder<VrfTables> idBuilder =
1909             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1910         return idBuilder.build();
1911     }
1912
1913     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1914         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1915                 .FLOWID_SEPARATOR + nextHop;
1916     }
1917
1918     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1919         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1920                 + tableMiss + FLOWID_PREFIX;
1921     }
1922
1923     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1924         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1925             .child(VrfTables.class, new VrfTablesKey(rd))
1926             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1927         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1928         if (vrfEntry.isPresent()) {
1929             return vrfEntry.get();
1930         }
1931         return null;
1932     }
1933
1934     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1935                                              final String vpnName,
1936                                              final VrfEntry vrfEntry) {
1937         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1938
1939         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1940         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1941
1942         if (targetDpns.isEmpty()) {
1943             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1944             return;
1945         }
1946
1947         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1948         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1949
1950         // delete from FIB
1951         //
1952         optNextHop.ifPresent(nextHop -> {
1953             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1954             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1955             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1956                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1957
1958             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1959             for (BigInteger dpId : targetDpns) {
1960                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1961                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1962
1963                 mdsalManager.removeFlow(dpId, flow);
1964             }
1965         });
1966
1967         // delete from LFIB
1968         //
1969         optLabel.ifPresent(label -> {
1970             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1971
1972             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1973                 for (BigInteger dpId : targetDpns) {
1974                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1975                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1976                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1977                             tx);
1978                 }
1979             }), LOG, "Error removing flows");
1980         });
1981     }
1982
1983     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1984             List<String> nextHopAddressList, LabelRouteInfo lri) {
1985         return lri != null && lri.getPrefix().equals(prefix)
1986                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1987     }
1988
1989     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1990         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1991             return true;
1992         }
1993
1994         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1995         if (prefix != null) {
1996             BigInteger prefixDpnId = prefix.getDpnId();
1997             if (dpnId.equals(prefixDpnId)) {
1998                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1999                         vrfEntry, dpnId);
2000                 return false;
2001             }
2002         }
2003         return true;
2004     }
2005 }