fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  42 import org.opendaylight.genius.mdsalutil.ActionInfo;
  43 import org.opendaylight.genius.mdsalutil.FlowEntity;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  48 import org.opendaylight.genius.mdsalutil.NWUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.UpgradeState;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  54 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  57 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  63 import org.opendaylight.genius.utils.ServiceIndex;
  64 import org.opendaylight.genius.utils.batching.SubTransaction;
  65 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  66 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  67 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  68 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  69 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  70 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  71 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  72 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  73 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  74 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 110 import org.slf4j.Logger;
 111 import org.slf4j.LoggerFactory;
 112
 113
 114 @Singleton
 115 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 118     private static final String FLOWID_PREFIX = "L3.";
 119     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 120     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 121     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 122     private static final int LFIB_INTERVPN_PRIORITY = 15;
 123     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 124     private static final int MAX_RETRIES = 3;
 125     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 126
 127     private final DataBroker dataBroker;
 128     private final ManagedNewTransactionRunner txRunner;
 129     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 130     private final IMdsalApiManager mdsalManager;
 131     private final NexthopManager nextHopManager;
 132     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 133     private final BaseVrfEntryHandler baseVrfEntryHandler;
 134     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 135     private final JobCoordinator jobCoordinator;
 136     private final IElanService elanManager;
 137     private final FibUtil fibUtil;
 138     private final InterVpnLinkCache interVpnLinkCache;
 139     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 140     private final UpgradeState upgradeState;
 141     private final DataTreeEventCallbackRegistrar eventCallbacks;
 142
 143     @Inject
 144     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 145                             final NexthopManager nexthopManager,
 146                             final IElanService elanManager,
 147                             final BaseVrfEntryHandler vrfEntryHandler,
 148                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 149                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 150                             final JobCoordinator jobCoordinator,
 151                             final FibUtil fibUtil,
 152                             final InterVpnLinkCache interVpnLinkCache,
 153                             final UpgradeState upgradeState,
 154                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 155         super(VrfEntry.class, VrfEntryListener.class);
 156         this.dataBroker = dataBroker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 158         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 159         this.mdsalManager = mdsalApiManager;
 160         this.nextHopManager = nexthopManager;
 161         this.elanManager = elanManager;
 162         this.baseVrfEntryHandler = vrfEntryHandler;
 163         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 164         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.fibUtil = fibUtil;
 167         this.interVpnLinkCache = interVpnLinkCache;
 168         this.upgradeState = upgradeState;
 169         this.eventCallbacks = eventCallbacks;
 170     }
 171
 172     @Override
 173     @PostConstruct
 174     public void init() {
 175         LOG.info("{} init", getClass().getSimpleName());
 176         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 177     }
 178
 179     @Override
 180     @SuppressWarnings("checkstyle:IllegalCatch")
 181     public void close() {
 182         closeables.forEach(c -> {
 183             try {
 184                 c.close();
 185             } catch (Exception e) {
 186                 LOG.warn("Error closing {}", c, e);
 187             }
 188         });
 189     }
 190
 191     @Override
 192     protected VrfEntryListener getDataTreeChangeListener() {
 193         return VrfEntryListener.this;
 194     }
 195
 196     @Override
 197     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 198         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 199     }
 200
 201     @Override
 202     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 203         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 204         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 205         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 206                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 207         addFibEntries(identifier, vrfEntry, rd);
 208         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 209                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 210     }
 211
 212     //This method is temporary. Eventually Factory design pattern will be used to get
 213     // right VrfEntryhandle and invoke its methods.
 214     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 215         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 216             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 217             return;
 218         }
 219         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 220             LOG.info("EVPN flows need to be programmed.");
 221             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 222                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 223             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 224             closeables.add(evpnVrfEntryHandler);
 225             return;
 226         }
 227         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 228         if (routerInt != null) {
 229             // ping responder for router interfaces
 230             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 234             createFibEntries(identifier, vrfEntry);
 235             return;
 236         }
 237     }
 238
 239     @Override
 240     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 241         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 242         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 243         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 244                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 245         removeFibEntries(identifier, vrfEntry, rd);
 246         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 247             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 248     }
 249
 250     //This method is temporary. Eventually Factory design pattern will be used to get
 251     // right VrfEntryhandle and invoke its methods.
 252     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 253         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 254             LOG.info("EVPN flows to be deleted");
 255             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 256                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 257             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 258             closeables.add(evpnVrfEntryHandler);
 259             return;
 260         }
 261         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 262         if (routerInt != null) {
 263             // ping responder for router interfaces
 264             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 265             return;
 266         }
 267         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 268             deleteFibEntries(identifier, vrfEntry);
 269             return;
 270         }
 271         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 272             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 273             return;
 274         }
 275     }
 276
 277     @Override
 278     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 279     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 280     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 281     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 282         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 283         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 284         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 285                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 286         // Handle BGP Routes first
 287         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 288             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 289             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 290                     rd, update.getDestPrefix(), update.getRoutePaths());
 291             return;
 292         }
 293
 294         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 295             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 296             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 297             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 298
 299             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 300             //updated vrf entry route path is null. This can happen during tunnel events.
 301             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 302             List<String> usedRds = new ArrayList<>();
 303             if (optVpnInstance.isPresent()) {
 304                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 305                         update.getDestPrefix());
 306             }
 307             // If original VRF Entry had nexthop null , but update VRF Entry
 308             // has nexthop , route needs to be created on remote Dpns
 309             if (originalRoutePath == null || originalRoutePath.isEmpty()
 310                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 311                 // TODO(vivek): Though ugly, Not handling this code now, as each
 312                 // tep add event will invoke flow addition
 313                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 314                         + " This event is IGNORED here.", update.getDestPrefix());
 315                 return;
 316             }
 317
 318             // If original VRF Entry had valid nexthop , but update VRF Entry
 319             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 320             if (updateRoutePath == null || updateRoutePath.isEmpty()
 321                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 322                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 323                         + "This event is IGNORED here.", update.getDestPrefix());
 324                 return;
 325             }
 326             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 327             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 328             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 329             ListenableFuture<Void> future =
 330                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> nextHopsRemoved.parallelStream()
 331                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 332                                     tx, nextHopRemoved, rd, update.getDestPrefix())));
 333             Futures.addCallback(future, new FutureCallback<Void>() {
 334                 @Override
 335                 public void onSuccess(Void result) {
 336                     createFibEntries(identifier, update);
 337                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 338                             rd, update.getDestPrefix(), update.getRoutePaths());
 339                 }
 340
 341                 @Override
 342                 public void onFailure(Throwable throwable) {
 343                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 344                             update, throwable);
 345                 }
 346             }, MoreExecutors.directExecutor());
 347             return;
 348         }
 349
 350         //Handle all other routes only on a cluster reboot
 351         if (original.equals(update)) {
 352             //Reboot use-case
 353             createFibEntries(identifier, update);
 354             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 355                     rd, update.getDestPrefix(), update.getRoutePaths());
 356             return;
 357         }
 358
 359         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 360                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 361     }
 362
 363     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 364         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 365         List<SubTransaction> txnObjects =  new ArrayList<>();
 366         final VpnInstanceOpDataEntry vpnInstance =
 367                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 368         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 369         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 370                 + " has null vpnId!");
 371         final Collection<VpnToDpnList> vpnToDpnList;
 372         if (vrfEntry.getParentVpnRd() != null
 373                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 374             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 375             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 376             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 377                 vpnInstance.getVpnToDpnList();
 378             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 379                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 380         } else {
 381             vpnToDpnList = vpnInstance.getVpnToDpnList();
 382         }
 383         final Long vpnId = vpnInstance.getVpnId();
 384         final String rd = vrfTableKey.getRouteDistinguisher();
 385         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 386         if (subnetRoute != null) {
 387             final long elanTag = subnetRoute.getElantag();
 388             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 389                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 390             if (vpnToDpnList != null) {
 391                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 392                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 393                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 394                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 395                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 396                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 397                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 398                             }
 399                         }
 400                     })));
 401             }
 402             return;
 403         }
 404         // Get etherType value based on the IpPrefix address family type
 405         int etherType;
 406         try {
 407             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 408         } catch (IllegalArgumentException ex) {
 409             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 410             return;
 411         }
 412
 413         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 414         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 415             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 416                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 417                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 418                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 419                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 420                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 421                                     try {
 422                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 423                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 424                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 425                                                     txnObjects);
 426                                         } else {
 427                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 428                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 429                                         }
 430                                     } catch (NullPointerException e) {
 431                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 432                                                 vrfEntry.getDestPrefix(), e);
 433                                     }
 434                                 }
 435                             }
 436                         }
 437                     }
 438                 })), MAX_RETRIES);
 439         }
 440
 441         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 442         if (optVpnUuid.isPresent()) {
 443             String vpnUuid = optVpnUuid.get();
 444             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 445             if (interVpnLink != null) {
 446                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 447                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 448                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 449                         // This is an static route that points to the other endpoint of an InterVpnLink
 450                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 451                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 452                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 453                     }
 454                 });
 455             }
 456         }
 457     }
 458
 459     void refreshFibTables(String rd, String prefix) {
 460         InstanceIdentifier<VrfEntry> vrfEntryId =
 461                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 462                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 463         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 464         if (vrfEntry.isPresent()) {
 465             createFibEntries(vrfEntryId, vrfEntry.get());
 466         }
 467     }
 468
 469     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 470         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 471         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 472         prefixBuilder.setDpnId(lri.getDpnId());
 473         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 474         prefixBuilder.setIpAddress(lri.getPrefix());
 475         // Increment the refCount here
 476         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 477             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 478         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 479         if (!isPresentInList) {
 480             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 481             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 482             vpnInstanceNames.add(vpnInstanceName);
 483             builder.setVpnInstanceList(vpnInstanceNames);
 484             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 485         } else {
 486             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 487         }
 488         return prefixBuilder.build();
 489     }
 490
 491     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 492                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 493         if (tx == null) {
 494             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 495                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 496                 "Error installing subnet route in FIB");
 497             return;
 498         }
 499         int etherType;
 500         try {
 501             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 502         } catch (IllegalArgumentException ex) {
 503             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 504             return;
 505         }
 506         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 507             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 508             synchronized (label.toString().intern()) {
 509                 LabelRouteInfo lri = getLabelRouteInfo(label);
 510                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 511
 512                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 513                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 514                                 fibUtil.getVpnInstanceOpData(rd);
 515                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 516                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 517                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 518                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 519                             }
 520                         }
 521                     }
 522                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 523                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 524                 }
 525             }
 526         });
 527         final List<InstructionInfo> instructions = new ArrayList<>();
 528         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 529             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 530         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 531         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 532         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 533                 NwConstants.ADD_FLOW, tx, null);
 534         if (vrfEntry.getRoutePaths() != null) {
 535             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 536                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 537                     List<ActionInfo> actionsInfos = new ArrayList<>();
 538                     // reinitialize instructions list for LFIB Table
 539                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 540                     actionsInfos.add(new ActionPopMpls(etherType));
 541                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 542                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 543                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 544                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 545
 546                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 547                             NwConstants.ADD_FLOW, tx);
 548                 }
 549             }
 550         }
 551     }
 552
 553     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 554                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 555         List<MatchInfo> matches = new ArrayList<>();
 556
 557         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 558                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 559         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 560         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 561         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 562
 563         InetAddress destPrefix;
 564         try {
 565             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 566         } catch (UnknownHostException e) {
 567             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 568                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 569             return;
 570         }
 571
 572         // Match on VpnId and SubnetBroadCast IP address
 573         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 574         matches.add(MatchEthernetType.IPV4);
 575
 576         if (prefixLength != 0) {
 577             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 578         }
 579
 580         //Action is to drop the packet
 581         List<InstructionInfo> dropInstructions = new ArrayList<>();
 582         List<ActionInfo> actionsInfos = new ArrayList<>();
 583         actionsInfos.add(new ActionDrop());
 584         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 585
 586         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 587         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 588         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 589                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 590
 591         Flow flow = flowEntity.getFlowBuilder().build();
 592         String flowId = flowEntity.getFlowId();
 593         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 594         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 595
 596         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 597                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 598                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 599
 600         if (addOrRemove == NwConstants.ADD_FLOW) {
 601             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 602         } else {
 603             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 604         }
 605     }
 606
 607     /*
 608      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 609      * LportDispatcher table (via table 80)
 610      */
 611     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 612                                             final VrfEntry vrfEntry, int etherType) {
 613         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 614         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 615         // packet is commuted from Vpn2 to Vpn1.
 616         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 617         if (!interVpnLink.isActive()) {
 618             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 619                      interVpnLinkName, vrfEntry.getDestPrefix());
 620             return;
 621         }
 622
 623         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 624         if (!optLportTag.isPresent()) {
 625             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 626             return;
 627         }
 628
 629         Long lportTag = optLportTag.get();
 630         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 631         if (label == null) {
 632             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 633                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 634             return;
 635         }
 636         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 637         List<InstructionInfo> instructions = Arrays.asList(
 638             new InstructionApplyActions(actionsInfos),
 639             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 640                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 641                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 642                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 643             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 644         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 645         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 646
 647         for (BigInteger dpId : targetDpns) {
 648             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 649                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 650
 651             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 652                                /*writeTx*/null);
 653         }
 654     }
 655
 656
 657     /*
 658      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 659      */
 660     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 661                                                final VrfEntry vrfEntry, long vpnTag) {
 662         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 663         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 664             && vrfEntry.getRoutePaths().size() == 1);
 665         String destination = vrfEntry.getDestPrefix();
 666         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 667         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 668
 669         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 670         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 671         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 672         if (interVpnLink.getState().or(State.Error) != State.Active) {
 673             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 674                 destination, nextHop, interVpnLinkName);
 675             return;
 676         }
 677
 678         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 679         if (!optOtherEndpointLportTag.isPresent()) {
 680             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 681                 vpnUuid, interVpnLinkName);
 682             return;
 683         }
 684
 685         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 686         if (targetDpns.isEmpty()) {
 687             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 688                 vpnUuid, interVpnLinkName);
 689             return;
 690         }
 691
 692         String[] values = destination.split("/");
 693         String destPrefixIpAddress = values[0];
 694         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 695
 696         List<MatchInfo> matches = new ArrayList<>();
 697         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 698         matches.add(MatchEthernetType.IPV4);
 699
 700         if (prefixLength != 0) {
 701             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 702         }
 703
 704         List<Instruction> instructions =
 705             Arrays.asList(new InstructionWriteMetadata(
 706                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 707                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 708                             .L3VPN_SERVICE_INDEX)),
 709                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 710                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 711
 712         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 713         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 714         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 715             COOKIE_VM_FIB_TABLE, matches, instructions);
 716
 717         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 718             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 719
 720         for (BigInteger dpId : targetDpns) {
 721
 722             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 723                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 724                 dpId, interVpnLink.getInterVpnLinkName());
 725
 726             mdsalManager.installFlow(dpId, flowEntity);
 727         }
 728     }
 729
 730     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 731         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 732         String localNextHopIP = vrfEntry.getDestPrefix();
 733         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 734         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 735         if (localNextHopInfo == null) {
 736             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 737             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 738                     vpnName, usedRds, localNextHopIP);
 739             boolean localNextHopSeen = false;
 740             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 741             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 742                 String ipPrefix;
 743                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 744                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 745                 } else {
 746                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 747                 }
 748                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 749                     ipPrefix);
 750                 if (localNextHopInfoLocal != null) {
 751                     localNextHopSeen = true;
 752                     BigInteger dpnId =
 753                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 754                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 755                     returnLocalDpnId.add(dpnId);
 756                 }
 757             }
 758             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 759                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 760                 if (optionalLabel.isPresent()) {
 761                     Long label = optionalLabel.get();
 762                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 763                     synchronized (label.toString().intern()) {
 764                         LabelRouteInfo lri = getLabelRouteInfo(label);
 765                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 766                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 767                                     fibUtil.getVpnInstanceOpData(rd);
 768                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 769                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 770                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 771                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 772                                     localNextHopIP = lri.getPrefix();
 773                                 } else {
 774                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 775                                     localNextHopIP = lri.getPrefix();
 776                                 }
 777                             }
 778                             if (localNextHopInfo != null) {
 779                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 780                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 781                                 if (vpnExtraRoutes.isEmpty()) {
 782                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 783                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 784                                     returnLocalDpnId.add(dpnId);
 785                                 } else {
 786                                     for (Routes extraRoutes : vpnExtraRoutes) {
 787                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 788                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 789                                         returnLocalDpnId.add(dpnId);
 790                                     }
 791                                 }
 792                             }
 793                         }
 794                     }
 795                 }
 796             }
 797             if (returnLocalDpnId.isEmpty()) {
 798                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 799             }
 800         } else {
 801             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 802                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 803             if (dpnId != null) {
 804                 returnLocalDpnId.add(dpnId);
 805             }
 806         }
 807         return returnLocalDpnId;
 808     }
 809
 810     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 811                                                 final Long vpnId, final String rd,
 812                                                 final VrfEntry vrfEntry,
 813                                                 Routes routes, List<Routes> vpnExtraRoutes,
 814                                                 int etherType) {
 815         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 816         if (localNextHopInfo != null) {
 817             long groupId;
 818             long localGroupId;
 819             final BigInteger dpnId = localNextHopInfo.getDpnId();
 820             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 821                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 822                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 823                 return dpnId;
 824             }
 825             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 826                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 827                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 828                 return dpnId;
 829             }
 830             if (!isVpnPresentInDpn(rd, dpnId)) {
 831                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 832                         vpnId, rd, dpnId.toString());
 833                 return BigInteger.ZERO;
 834             }
 835             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 836             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 837             String prefix = vrfEntry.getDestPrefix();
 838             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 839             //The loadbalancing group is created only if the extra route has multiple nexthops
 840             //to avoid loadbalancing the discovered routes
 841             if (vpnExtraRoutes != null && routes != null) {
 842                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 843                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 844                 } else {
 845                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 846                 }
 847                 if (vpnExtraRoutes.size() > 1) {
 848                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 849                             vpnExtraRoutes);
 850                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 851                 } else if (routes.getNexthopIpList().size() > 1) {
 852                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes,
 853                             vpnExtraRoutes);
 854                     localGroupId = groupId;
 855                 } else {
 856                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 857                             prefix, gwMacAddress, jobKey);
 858                     localGroupId = groupId;
 859                 }
 860             } else {
 861                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 862                         gwMacAddress, jobKey);
 863                 localGroupId = groupId;
 864             }
 865             if (groupId == FibConstants.INVALID_GROUP_ID) {
 866                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 867                         prefix, rd, interfaceName, dpnId.toString());
 868                 return BigInteger.ZERO;
 869             }
 870             final List<InstructionInfo> instructions = Collections.singletonList(
 871                     new InstructionApplyActions(
 872                             Collections.singletonList(new ActionGroup(groupId))));
 873             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 874                     new InstructionApplyActions(
 875                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(groupId))));
 876             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 877             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 878             jobCoordinator.enqueueJob(jobKey, () -> {
 879                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 880                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 881                             NwConstants.ADD_FLOW, tx, null);
 882                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 883                             vpnName, rd)) {
 884                         optLabel.ifPresent(label -> {
 885                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 886                                 LOG.debug(
 887                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 888                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 889                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 890                                         nextHopAddressList);
 891                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 892                                         NwConstants.ADD_FLOW, tx);
 893                                 // If the extra-route is reachable from VMs attached to the same switch,
 894                                 // then the tunnel table can point to the load balancing group.
 895                                 // If it is reachable from VMs attached to different switches,
 896                                 // then it should be pointing to one of the local group in order to avoid looping.
 897                                 if (vrfEntry.getRoutePaths().size() == 1) {
 898                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 899                                 } else {
 900                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 901                                 }
 902                             } else {
 903                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 904                                                 + "route. LFib and Terminating table entries will not be created.",
 905                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 906                             }
 907                         });
 908                     }
 909                 }));
 910             });
 911             return dpnId;
 912         }
 913         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 914                 vpnName);
 915         return BigInteger.ZERO;
 916     }
 917
 918     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 919         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 920         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 921         if (dpnInVpn.isPresent()) {
 922             return true;
 923         }
 924         return false;
 925     }
 926
 927     private LabelRouteInfo getLabelRouteInfo(Long label) {
 928         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 929             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 930         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 931         if (opResult.isPresent()) {
 932             return opResult.get();
 933         }
 934         return null;
 935     }
 936
 937     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 938         if (lri == null) {
 939             return true;
 940         }
 941
 942         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 943         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 944             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 945
 946         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 947             ? lri.getVpnInstanceList() : new ArrayList<>();
 948         if (vpnInstancesList.contains(vpnInstanceName)) {
 949             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 950             vpnInstancesList.remove(vpnInstanceName);
 951         }
 952         if (vpnInstancesList.isEmpty()) {
 953             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 954             if (tx != null) {
 955                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 956             } else {
 957                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 958             }
 959             return true;
 960         } else {
 961             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 962             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 963             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 964         }
 965         return false;
 966     }
 967
 968     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 969                                       WriteTransaction tx) {
 970         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 971
 972         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 973
 974         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 975             dpId, label, groupId);
 976     }
 977
 978     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 979                                                 WriteTransaction tx) {
 980         List<MatchInfo> mkMatches = new ArrayList<>();
 981
 982         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 983             destDpId, label, actionsInfos);
 984
 985         // Matching metadata
 986         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 987         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 988
 989         List<InstructionInfo> mkInstructions = new ArrayList<>();
 990         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 991
 992         FlowEntity terminatingServiceTableFlowEntity =
 993             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 994             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 995                 String.format("%s:%d", "TST Flow Entry ", label),
 996             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 997
 998         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 999
1000         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1001
1002         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1003         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1004             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1005             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1006             .child(Flow.class, flowKey).build();
1007         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
1008                 WriteTransaction.CREATE_MISSING_PARENTS);
1009     }
1010
1011     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1012         FlowEntity flowEntity;
1013         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1014         List<MatchInfo> mkMatches = new ArrayList<>();
1015         // Matching metadata
1016         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1017         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1018             NwConstants.INTERNAL_TUNNEL_TABLE,
1019             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1020             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1021             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1022         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1023         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1024         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1025             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1026             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1027
1028         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1029         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1030     }
1031
1032     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1033         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1034         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1035         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1036         boolean shouldUpdateNonEcmpLocalNextHop = true;
1037         if (localNextHopInfo == null) {
1038             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1039             if (usedRds.size() > 1) {
1040                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1041                         vrfEntry.getDestPrefix(), vpnName, rd);
1042                 return returnLocalDpnId;
1043             }
1044             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1045             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1046             //in the vpn
1047             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1048                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1049             if (extraRouteOptional.isPresent()) {
1050                 Routes extraRoute = extraRouteOptional.get();
1051                 String ipPrefix;
1052                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1053                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1054                 } else {
1055                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1056                 }
1057                 if (extraRoute.getNexthopIpList().size() > 1) {
1058                     shouldUpdateNonEcmpLocalNextHop = false;
1059                 }
1060                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1061                 if (localNextHopInfo != null) {
1062                     String localNextHopIP = localNextHopInfo.getIpAddress();
1063                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1064                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1065                     if (!dpnId.equals(BigInteger.ZERO)) {
1066                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1067                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1068                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1069                                 /*remove*/ false);
1070                         returnLocalDpnId.add(dpnId);
1071                     }
1072                 } else {
1073                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1074                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1075                 }
1076             }
1077
1078             if (localNextHopInfo == null) {
1079                 /* Imported VRF entry */
1080                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1081                 if (optionalLabel.isPresent()) {
1082                     Long label = optionalLabel.get();
1083                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1084                     LabelRouteInfo lri = getLabelRouteInfo(label);
1085                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1086                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1087                         prefixBuilder.setDpnId(lri.getDpnId());
1088                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1089                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1090                         if (!dpnId.equals(BigInteger.ZERO)) {
1091                             returnLocalDpnId.add(dpnId);
1092                         }
1093                     }
1094                 }
1095             }
1096
1097         } else {
1098             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1099             String localNextHopIP = localNextHopInfo.getIpAddress();
1100             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1101                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1102             if (!dpnId.equals(BigInteger.ZERO)) {
1103                 returnLocalDpnId.add(dpnId);
1104             }
1105         }
1106
1107         return returnLocalDpnId;
1108     }
1109
1110     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1111                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1112                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1113         if (localNextHopInfo != null) {
1114             final BigInteger dpnId = localNextHopInfo.getDpnId();
1115             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1116                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1117                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1118                 return dpnId;
1119             }
1120             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1121                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1122                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1123                 return dpnId;
1124             }
1125
1126             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1127                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1128                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1129                             NwConstants.DEL_FLOW, tx, null);
1130                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1131                             vpnId, rd)) {
1132                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1133                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1134                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1135                                         NwConstants.DEL_FLOW, tx);
1136                                 removeTunnelTableEntry(dpnId, label, tx);
1137                             });
1138                         }
1139                     }
1140                 })));
1141             //TODO: verify below adjacency call need to be optimized (?)
1142             //In case of the removal of the extra route, the loadbalancing group is updated
1143             if (shouldUpdateNonEcmpLocalNextHop) {
1144                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1145             }
1146             return dpnId;
1147         }
1148         return BigInteger.ZERO;
1149     }
1150
1151     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1152             final VrfEntry vrfEntry, WriteTransaction tx) {
1153         if (tx == null) {
1154             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1155                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1156             }), LOG, "Error creating remote FIB entry");
1157             return;
1158         }
1159
1160         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1161         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1162                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1163
1164         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1165         if (adjacencyResults.isEmpty()) {
1166             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1167                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1168             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1169             return;
1170         }
1171
1172         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1173         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1174                 vpnName, usedRds, vrfEntry.getDestPrefix());
1175         // create loadbalancing groups for extra routes only when the extra route is present behind
1176         // multiple VMs
1177         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1178                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1179             List<InstructionInfo> instructions = new ArrayList<>();
1180             // Obtain the local routes for this particular dpn.
1181             java.util.Optional<Routes> routes = vpnExtraRoutes
1182                     .stream()
1183                     .filter(route -> {
1184                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1185                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1186                         if (prefixToInterface == null) {
1187                             return false;
1188                         }
1189                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1190                     }).findFirst();
1191             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1192                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1193             if (groupId == FibConstants.INVALID_GROUP_ID) {
1194                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1195                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1196                 return;
1197             }
1198             List<ActionInfo> actionInfos =
1199                     Collections.singletonList(new ActionGroup(groupId));
1200             instructions.add(new InstructionApplyActions(actionInfos));
1201             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1202                     NwConstants.ADD_FLOW, tx, null);
1203         } else {
1204             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1205         }
1206
1207         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1208     }
1209
1210     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1211     /* Get interface info from prefix to interface mapping;
1212         Use the interface info to get the corresponding vpn interface op DS entry,
1213         remove the adjacency corresponding to this fib entry.
1214         If adjacency removed is the last adjacency, clean up the following:
1215          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1216          - prefix to interface entry
1217          - vpn interface op DS
1218      */
1219         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1220         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1221         if (prefixInfo == null) {
1222             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1223             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1224             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1225             if (extraRoute != null) {
1226                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1227                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1228                     if (nextHopIp != null) {
1229                         String ipPrefix;
1230                         if (isIpv4Address(nextHopIp)) {
1231                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1232                         } else {
1233                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1234                         }
1235                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1236                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1237                     }
1238                 }
1239             }
1240             if (prefixInfo == null) {
1241                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1242                 if (optionalLabel.isPresent()) {
1243                     Long label = optionalLabel.get();
1244                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1245                     LabelRouteInfo lri = getLabelRouteInfo(label);
1246                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1247                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1248                         prefixBuilder.setDpnId(lri.getDpnId());
1249                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1250                         prefixBuilder.setIpAddress(lri.getPrefix());
1251                         prefixInfo = prefixBuilder.build();
1252                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1253                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1254                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1255                     }
1256                 }
1257             }
1258         } else {
1259             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1260         }
1261     }
1262
1263     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1264                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1265
1266         if (prefixInfo == null) {
1267             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1268                     vrfEntry.getDestPrefix(), vpnId, rd);
1269             return; //Don't have any info for this prefix (shouldn't happen); need to return
1270         }
1271
1272         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1273             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1274                     vrfEntry.getDestPrefix(), vpnId, rd);
1275             return;
1276         }
1277
1278         String ifName = prefixInfo.getVpnInterfaceName();
1279         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1280             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1281     }
1282
1283     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1284         Prefixes prefixInfo;
1285         Long vpnId;
1286         String rd;
1287         VrfEntry vrfEntry;
1288         Routes extraRoute;
1289
1290         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1291                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1292             this.prefixInfo = prefixInfo;
1293             this.vpnId = vpnId;
1294             this.rd = rd;
1295             this.vrfEntry = vrfEntry;
1296             this.extraRoute = extraRoute;
1297         }
1298
1299         @Override
1300         public List<ListenableFuture<Void>> call() {
1301             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1302             // to call the respective helpers.
1303             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1304                 //First Cleanup LabelRouteInfo
1305                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1306                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1307                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1308                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1309                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1310                         synchronized (label.toString().intern()) {
1311                             LabelRouteInfo lri = getLabelRouteInfo(label);
1312                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1313                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1314                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1315                                         fibUtil.getVpnInstanceOpData(rd);
1316                                 String vpnInstanceName = "";
1317                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1318                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1319                                 }
1320                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1321                                 if (lriRemoved) {
1322                                     String parentRd = lri.getParentVpnRd();
1323                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1324                                             parentRd, vrfEntry.getDestPrefix()));
1325                                 }
1326                             } else {
1327                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1328                                         rd, vrfEntry.getDestPrefix()));
1329                             }
1330                         }
1331                     });
1332                 }
1333                 String ifName = prefixInfo.getVpnInterfaceName();
1334                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1335                 String vpnName = null;
1336
1337                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1338                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1339                     * */
1340                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1341                     if (vpnNameOpt.isPresent()) {
1342                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1343                     }
1344                 }
1345                 if (optVpnName.isPresent()) {
1346                     vpnName = optVpnName.get();
1347                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1348                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1349                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1350                     if (opVpnInterface.isPresent()) {
1351                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1352                         if (vpnId != associatedVpnId) {
1353                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1354                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1355                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1356                             return;
1357                         } else {
1358                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1359                                     vrfEntry.getDestPrefix(), associatedVpnId);
1360                         }
1361                     }
1362                 }
1363                 if (extraRoute != null) {
1364                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1365                     //Only one used Rd present in case of removal event
1366                     String usedRd = usedRds.get(0);
1367                     if (optVpnName.isPresent()) {
1368                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1369                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1370                                         vrfEntry.getDestPrefix()));
1371                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1372                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1373                     }
1374                 }
1375                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1376             }));
1377         }
1378     }
1379
1380     /**
1381      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1382      * Remove Adjacency from VPNInterfaceOpData.
1383      * if Adjacency != primary.
1384      * if Adjacency == primary , then mark it for deletion.
1385      * Remove entire VPNinterfaceOpData Entry.
1386      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1387      * @param vrfEntry - VrfEntry removed
1388      * @param ifName - Interface name from VRFentry
1389      * @param vpnName - VPN name of corresponding VRF
1390      * @param tx - ReadWrite Tx
1391      * @throws ReadFailedException - Exception thrown in case of read failed
1392      */
1393     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1394                                                           ReadWriteTransaction tx) throws ReadFailedException {
1395         InstanceIdentifier<Adjacency> adjacencyIid =
1396                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1397         Optional<Adjacency> adjacencyOptional = tx.read(LogicalDatastoreType.OPERATIONAL, adjacencyIid).checkedGet();
1398         if (adjacencyOptional.isPresent()) {
1399             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1400                 tx.delete(LogicalDatastoreType.OPERATIONAL,
1401                         FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1402             } else {
1403                 tx.merge(LogicalDatastoreType.OPERATIONAL, adjacencyIid,
1404                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1405             }
1406         }
1407
1408         Optional<AdjacenciesOp> optAdjacencies =
1409                 tx.read(LogicalDatastoreType.OPERATIONAL,
1410                         FibUtil.getAdjListPathOp(ifName, vpnName)).checkedGet();
1411
1412         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1413             return;
1414         }
1415
1416         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1417                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1418                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1419                         && adjacency.isMarkedForDeletion() != null
1420                         && adjacency.isMarkedForDeletion()
1421         )) {
1422             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1423             tx.delete(LogicalDatastoreType.OPERATIONAL,
1424                     FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1425         }
1426     }
1427
1428     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1429         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1430         final String rd = vrfTableKey.getRouteDistinguisher();
1431         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1432         if (vpnInstance == null) {
1433             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1434             return;
1435         }
1436         final Collection<VpnToDpnList> vpnToDpnList;
1437         if (vrfEntry.getParentVpnRd() != null
1438                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1439             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1440             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1441             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1442                     vpnInstance.getVpnToDpnList();
1443             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1444                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1445         } else {
1446             vpnToDpnList = vpnInstance.getVpnToDpnList();
1447         }
1448
1449         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1450         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1451         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1452         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1453         if (subnetRoute != null) {
1454             long elanTag = subnetRoute.getElantag();
1455             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1456                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1457             if (vpnToDpnList != null) {
1458                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1459                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1460                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1461
1462                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1463                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1464                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1465                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1466                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1467                             }
1468
1469                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1470                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1471                         }
1472                     })));
1473             }
1474             optionalLabel.ifPresent(label -> {
1475                 synchronized (label.toString().intern()) {
1476                     LabelRouteInfo lri = getLabelRouteInfo(label);
1477                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1478                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1479                                 fibUtil.getVpnInstanceOpData(rd);
1480                         String vpnInstanceName = "";
1481                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1482                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1483                         }
1484                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1485                         if (lriRemoved) {
1486                             String parentRd = lri.getParentVpnRd();
1487                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1488                                     parentRd, vrfEntry.getDestPrefix()));
1489                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1490                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1491                         }
1492                     } else {
1493                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1494                                 rd, vrfEntry.getDestPrefix()));
1495                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1496                                 label, rd, vrfEntry.getDestPrefix());
1497                     }
1498                 }
1499             });
1500             return;
1501         }
1502
1503         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1504             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1505         if (vpnToDpnList != null) {
1506             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1507                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1508             String jobKey;
1509             Optional<Routes> extraRouteOptional;
1510             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1511             if (usedRds != null && !usedRds.isEmpty()) {
1512                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1513                 if (usedRds.size() > 1) {
1514                     LOG.error("The extra route prefix is still present in some DPNs");
1515                     return ;
1516                 } else {
1517                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1518                     //is not present in any other DPN
1519                     extraRouteOptional = VpnExtraRouteHelper
1520                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1521                 }
1522             } else {
1523                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1524                 extraRouteOptional = Optional.absent();
1525             }
1526
1527             jobCoordinator.enqueueJob(jobKey,
1528                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1529                     if (localDpnIdList.size() <= 0) {
1530                         for (VpnToDpnList curDpn : vpnToDpnList) {
1531                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1532                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1533                         }
1534                     } else {
1535                         for (BigInteger localDpnId : localDpnIdList) {
1536                             for (VpnToDpnList curDpn : vpnToDpnList) {
1537                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1538                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1539                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1540                                 }
1541                             }
1542                         }
1543                     }
1544                 })), MAX_RETRIES);
1545         }
1546
1547         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1548         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1549         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1550
1551         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1552         // of the interVpnLink.
1553         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1554         if (optVpnUuid.isPresent()) {
1555             String vpnUuid = optVpnUuid.get();
1556             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1557                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1558                 if (optInterVpnLink.isPresent()) {
1559                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1560                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1561                         // This is route that points to the other endpoint of an InterVpnLink
1562                         // In that case, we should look for the FIB table pointing to
1563                         // LPortDispatcher table and remove it.
1564                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1565                     }
1566                 }
1567             });
1568         }
1569
1570     }
1571
1572     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1573                                     int addOrRemove, WriteTransaction tx) {
1574         if (tx == null) {
1575             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1576                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1577                 "Error making LFIB table entry");
1578             return;
1579         }
1580
1581         List<MatchInfo> matches = new ArrayList<>();
1582         matches.add(MatchEthernetType.MPLS_UNICAST);
1583         matches.add(new MatchMplsLabel(label));
1584
1585         // Install the flow entry in L3_LFIB_TABLE
1586         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1587
1588         FlowEntity flowEntity;
1589         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1590             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1591         Flow flow = flowEntity.getFlowBuilder().build();
1592         String flowId = flowEntity.getFlowId();
1593         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1594         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1595         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1596             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1597             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1598
1599         if (addOrRemove == NwConstants.ADD_FLOW) {
1600             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1601         } else {
1602             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1603         }
1604
1605         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1606             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1607     }
1608
1609     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1610                                     final FutureCallback<List<Void>> callback) {
1611         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1612         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1613             () -> {
1614                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1615                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1616                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1617                         LogicalDatastoreType.CONFIGURATION, id);
1618                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1619                 if (!vrfTable.isPresent()) {
1620                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1621                     if (callback != null) {
1622                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1623                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1624                     }
1625                     return futures;
1626                 }
1627                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1628                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1629                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1630                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1631                             if (subnetRoute != null) {
1632                                 long elanTag = subnetRoute.getElantag();
1633                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1634                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1635                                         tx);
1636                                 continue;
1637                             }
1638                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1639                             if (routerInt != null) {
1640                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1641                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1642                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1643                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1644                                         NwConstants.ADD_FLOW);
1645                                 continue;
1646                             }
1647                             //Handle local flow creation for imports
1648                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1649                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1650                                 if (optionalLabel.isPresent()) {
1651                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1652                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1653                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1654                                         if (lri.getDpnId().equals(dpnId)) {
1655                                             try {
1656                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1657                                                         vrfEntry.getDestPrefix());
1658                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1659                                             } catch (IllegalArgumentException ex) {
1660                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1661                                                         vrfEntry.getDestPrefix());
1662                                             }
1663                                             continue;
1664                                         }
1665                                     }
1666                                 }
1667                             }
1668                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1669                                     vrfEntry, dpnId);
1670                             if (shouldCreateRemoteFibEntry) {
1671                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1672                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1673                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1674                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1675                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1676                                 } else {
1677                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1678                                             vrfEntry, tx);
1679                                 }
1680                             }
1681                         }
1682                     }));
1683                     if (callback != null) {
1684                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1685                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1686                     }
1687                 }
1688                 return futures;
1689             });
1690     }
1691
1692     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1693                                             final String localNextHopIp, final String remoteNextHopIp) {
1694         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1695             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1696         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1697         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1698         List<SubTransaction> txnObjects =  new ArrayList<>();
1699         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1700         if (vrfTable.isPresent()) {
1701             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1702                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1703                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1704                         vrfTable.get().getVrfEntry().stream()
1705                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1706                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1707                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1708                     }
1709                 })));
1710         }
1711     }
1712
1713     public void manageRemoteRouteOnDPN(final boolean action,
1714                                        final BigInteger localDpnId,
1715                                        final long vpnId,
1716                                        final String rd,
1717                                        final String destPrefix,
1718                                        final String destTepIp,
1719                                        final long label) {
1720         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1721
1722         if (vpnInstance == null) {
1723             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1724             return;
1725         }
1726
1727         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1728             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1729                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1730                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1731                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1732                     if (vrfEntry == null) {
1733                         return;
1734                     }
1735                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1736                             action, localDpnId, vpnId, rd, destPrefix);
1737                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1738                     VrfEntry modVrfEntry;
1739                     if (routePathList == null || routePathList.isEmpty()) {
1740                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1741                                 Collections.singletonList(destTepIp),
1742                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1743                     } else {
1744                         modVrfEntry = vrfEntry;
1745                     }
1746
1747                     if (action) {
1748                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1749                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1750                                 modVrfEntry, tx);
1751                     } else {
1752                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1753                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1754                                 vrfEntry.getDestPrefix());
1755                         if (usedRds.size() > 1) {
1756                             LOG.debug("The extra route prefix is still present in some DPNs");
1757                             return;
1758                         }
1759                         //Is this fib route an extra route? If yes, get the nexthop which would be
1760                         //an adjacency in the vpn
1761                         Optional<Routes> extraRouteOptional = Optional.absent();
1762                         if (usedRds.size() != 0) {
1763                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1764                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1765                                     usedRds.get(0), vrfEntry.getDestPrefix());
1766                         }
1767                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1768                                 extraRouteOptional, tx);
1769                     }
1770                 }
1771             })));
1772     }
1773
1774     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1775                                  final FutureCallback<List<Void>> callback) {
1776         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1777         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1778             () -> {
1779                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1780                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1781                 List<SubTransaction> txnObjects =  new ArrayList<>();
1782                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1783                         LogicalDatastoreType.CONFIGURATION, id);
1784                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1785                 if (vrfTable.isPresent()) {
1786                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1787                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1788                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1789                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1790                                 /* Handle subnet routes here */
1791                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1792                                 if (subnetRoute != null && !fibUtil
1793                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1794                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1795                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1796                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1797                                             NwConstants.DEL_FLOW, tx, null);
1798                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1799                                     if (routePaths != null) {
1800                                         for (RoutePaths routePath : routePaths) {
1801                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1802                                                     DEFAULT_FIB_FLOW_PRIORITY,
1803                                                     NwConstants.DEL_FLOW, tx);
1804                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1805                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1806                                                     vrfEntry.getDestPrefix());
1807                                         }
1808                                     }
1809                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1810                                             NwConstants.DEL_FLOW, tx);
1811                                     continue;
1812                                 }
1813                                 // ping responder for router interfaces
1814                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1815                                 if (routerInt != null) {
1816                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1817                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1818                                             routerInt.getMacAddress());
1819                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1820                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1821                                             NwConstants.DEL_FLOW);
1822                                     continue;
1823                                 }
1824
1825                                 //Handle local flow deletion for imports
1826                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1827                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1828                                     if (optionalLabel.isPresent()) {
1829                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1830                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1831                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1832                                                 lri) && lri.getDpnId().equals(dpnId)) {
1833                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1834                                         }
1835                                     }
1836                                 }
1837
1838                                 // Passing null as we don't know the dpn
1839                                 // to which prefix is attached at this point
1840                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1841                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1842                                 Optional<Routes> extraRouteOptional;
1843                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1844                                 //an adjacency in the vpn
1845                                 if (usedRds != null && !usedRds.isEmpty()) {
1846                                     if (usedRds.size() > 1) {
1847                                         LOG.error("The extra route prefix is still present in some DPNs");
1848                                         return;
1849                                     } else {
1850                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1851                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1852
1853                                     }
1854                                 } else {
1855                                     extraRouteOptional = Optional.absent();
1856                                 }
1857                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1858                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1859                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1860                                 } else {
1861                                     if (subnetRoute == null || !fibUtil
1862                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1863                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1864                                                 vrfTable.get().key(), vrfEntry, extraRouteOptional, tx);
1865                                     }
1866                                 }
1867                             }
1868                         }));
1869                     }
1870                     if (callback != null) {
1871                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1872                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1873                     }
1874                 } else {
1875                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1876                 }
1877                 return futures;
1878             });
1879
1880     }
1881
1882     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1883                                            final String localNextHopIp, final String remoteNextHopIp) {
1884         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1885                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1886             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1887         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1888         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1889         List<SubTransaction> txnObjects =  new ArrayList<>();
1890         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1891         if (vrfTable.isPresent()) {
1892             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1893                 () -> {
1894                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1895                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1896                             tx -> vrfTable.get().getVrfEntry().stream()
1897                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1898                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1899                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1900                     }
1901                 });
1902         }
1903     }
1904
1905     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1906         InstanceIdentifierBuilder<VrfTables> idBuilder =
1907             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1908         return idBuilder.build();
1909     }
1910
1911     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1912         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1913                 .FLOWID_SEPARATOR + nextHop;
1914     }
1915
1916     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1917         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1918                 + tableMiss + FLOWID_PREFIX;
1919     }
1920
1921     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1922         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1923             .child(VrfTables.class, new VrfTablesKey(rd))
1924             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1925         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1926         if (vrfEntry.isPresent()) {
1927             return vrfEntry.get();
1928         }
1929         return null;
1930     }
1931
1932     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1933                                              final String vpnName,
1934                                              final VrfEntry vrfEntry) {
1935         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1936
1937         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1938         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1939
1940         if (targetDpns.isEmpty()) {
1941             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1942             return;
1943         }
1944
1945         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1946         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1947
1948         // delete from FIB
1949         //
1950         optNextHop.ifPresent(nextHop -> {
1951             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1952             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1953             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1954                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1955
1956             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1957             for (BigInteger dpId : targetDpns) {
1958                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1959                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1960
1961                 mdsalManager.removeFlow(dpId, flow);
1962             }
1963         });
1964
1965         // delete from LFIB
1966         //
1967         optLabel.ifPresent(label -> {
1968             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1969
1970             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1971                 for (BigInteger dpId : targetDpns) {
1972                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1973                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1974                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1975                             tx);
1976                 }
1977             }), LOG, "Error removing flows");
1978         });
1979     }
1980
1981     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1982             List<String> nextHopAddressList, LabelRouteInfo lri) {
1983         return lri != null && lri.getPrefix().equals(prefix)
1984                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1985     }
1986
1987     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1988         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1989             return true;
1990         }
1991
1992         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1993         if (prefix != null) {
1994             BigInteger prefixDpnId = prefix.getDpnId();
1995             if (dpnId.equals(prefixDpnId)) {
1996                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1997                         vrfEntry, dpnId);
1998                 return false;
1999             }
2000         }
2001         return true;
2002     }
2003 }