fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.net.InetAddress;
  25 import java.net.UnknownHostException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.List;
  31 import java.util.Objects;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.locks.ReentrantLock;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.inject.Inject;
  39 import javax.inject.Singleton;
  40 import org.eclipse.jdt.annotation.NonNull;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.Datastore.Operational;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.TransactionAdapter;
  51 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  52 import org.opendaylight.genius.infra.TypedWriteTransaction;
  53 import org.opendaylight.genius.mdsalutil.ActionInfo;
  54 import org.opendaylight.genius.mdsalutil.FlowEntity;
  55 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  56 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  57 import org.opendaylight.genius.mdsalutil.MatchInfo;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NWUtil;
  60 import org.opendaylight.genius.mdsalutil.NwConstants;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  73 import org.opendaylight.genius.utils.JvmGlobalLocks;
  74 import org.opendaylight.genius.utils.ServiceIndex;
  75 import org.opendaylight.genius.utils.batching.SubTransaction;
  76 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  77 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  78 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  79 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  80 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  81 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  85 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  86 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.slf4j.Logger;
 123 import org.slf4j.LoggerFactory;
 124
 125
 126 @Singleton
 127 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 128
 129     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 130     private static final String FLOWID_PREFIX = "L3.";
 131     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 132     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 133     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 134     private static final int LFIB_INTERVPN_PRIORITY = 15;
 135     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 136     private static final int MAX_RETRIES = 3;
 137     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 138
 139     private final DataBroker dataBroker;
 140     private final ManagedNewTransactionRunner txRunner;
 141     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 142     private final IMdsalApiManager mdsalManager;
 143     private final NexthopManager nextHopManager;
 144     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 145     private final BaseVrfEntryHandler baseVrfEntryHandler;
 146     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 147     private final JobCoordinator jobCoordinator;
 148     private final IElanService elanManager;
 149     private final FibUtil fibUtil;
 150     private final InterVpnLinkCache interVpnLinkCache;
 151     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 152     private final UpgradeState upgradeState;
 153     private final DataTreeEventCallbackRegistrar eventCallbacks;
 154
 155     @Inject
 156     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 157                             final NexthopManager nexthopManager,
 158                             final IElanService elanManager,
 159                             final BaseVrfEntryHandler vrfEntryHandler,
 160                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 161                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 162                             final JobCoordinator jobCoordinator,
 163                             final FibUtil fibUtil,
 164                             final InterVpnLinkCache interVpnLinkCache,
 165                             final UpgradeState upgradeState,
 166                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 167         super(VrfEntry.class, VrfEntryListener.class);
 168         this.dataBroker = dataBroker;
 169         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 170         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 171         this.mdsalManager = mdsalApiManager;
 172         this.nextHopManager = nexthopManager;
 173         this.elanManager = elanManager;
 174         this.baseVrfEntryHandler = vrfEntryHandler;
 175         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 176         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 177         this.jobCoordinator = jobCoordinator;
 178         this.fibUtil = fibUtil;
 179         this.interVpnLinkCache = interVpnLinkCache;
 180         this.upgradeState = upgradeState;
 181         this.eventCallbacks = eventCallbacks;
 182     }
 183
 184     @Override
 185     @PostConstruct
 186     public void init() {
 187         LOG.info("{} init", getClass().getSimpleName());
 188         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 189     }
 190
 191     @Override
 192     @SuppressWarnings("checkstyle:IllegalCatch")
 193     public void close() {
 194         closeables.forEach(c -> {
 195             try {
 196                 c.close();
 197             } catch (Exception e) {
 198                 LOG.warn("Error closing {}", c, e);
 199             }
 200         });
 201     }
 202
 203     @Override
 204     protected VrfEntryListener getDataTreeChangeListener() {
 205         return VrfEntryListener.this;
 206     }
 207
 208     @Override
 209     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 210         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 211     }
 212
 213     @Override
 214     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 215         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 216         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 217         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 218                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 219         addFibEntries(identifier, vrfEntry, rd);
 220         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 221                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 222     }
 223
 224     //This method is temporary. Eventually Factory design pattern will be used to get
 225     // right VrfEntryhandle and invoke its methods.
 226     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 227         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 228             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 229             return;
 230         }
 231         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 232             LOG.info("EVPN flows need to be programmed.");
 233             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 234                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 235             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 236             closeables.add(evpnVrfEntryHandler);
 237             return;
 238         }
 239         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 240         if (routerInt != null) {
 241             // ping responder for router interfaces
 242             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 243             return;
 244         }
 245         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 246             createFibEntries(identifier, vrfEntry);
 247             return;
 248         }
 249     }
 250
 251     @Override
 252     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 253         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 254         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 255         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 256                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 257         removeFibEntries(identifier, vrfEntry, rd);
 258         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 259             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 260     }
 261
 262     //This method is temporary. Eventually Factory design pattern will be used to get
 263     // right VrfEntryhandle and invoke its methods.
 264     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 265         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 266             LOG.info("EVPN flows to be deleted");
 267             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 268                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 269             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 270             closeables.add(evpnVrfEntryHandler);
 271             return;
 272         }
 273         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 274         if (routerInt != null) {
 275             // ping responder for router interfaces
 276             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 277             return;
 278         }
 279         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 280             deleteFibEntries(identifier, vrfEntry);
 281             return;
 282         }
 283         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 284             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 285             return;
 286         }
 287     }
 288
 289     @Override
 290     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 291     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 292     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 293     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 294         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 295         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 296         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 297                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 298         // Handle BGP Routes first
 299         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 300             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 301             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 302                     rd, update.getDestPrefix(), update.getRoutePaths());
 303             return;
 304         }
 305
 306         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 307             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 308             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 309             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 310
 311             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 312             //updated vrf entry route path is null. This can happen during tunnel events.
 313             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 314             List<String> usedRds = new ArrayList<>();
 315             if (optVpnInstance.isPresent()) {
 316                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 317                         update.getDestPrefix());
 318             }
 319             // If original VRF Entry had nexthop null , but update VRF Entry
 320             // has nexthop , route needs to be created on remote Dpns
 321             if (originalRoutePath == null || originalRoutePath.isEmpty()
 322                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 323                 // TODO(vivek): Though ugly, Not handling this code now, as each
 324                 // tep add event will invoke flow addition
 325                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 326                         + " This event is IGNORED here.", update.getDestPrefix());
 327                 return;
 328             }
 329
 330             // If original VRF Entry had valid nexthop , but update VRF Entry
 331             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 332             if (updateRoutePath == null || updateRoutePath.isEmpty()
 333                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 334                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 335                         + "This event is IGNORED here.", update.getDestPrefix());
 336                 return;
 337             }
 338             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 339             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 340             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 341             List<ListenableFuture<Void>> futures = new ArrayList<>();
 342             ListenableFuture<Void> configFuture =
 343                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 344                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 345                         nextHopsRemoved.parallelStream()
 346                             .forEach(nextHopRemoved -> {
 347                                 try {
 348                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 349                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 350                                 } catch (ExecutionException | InterruptedException e) {
 351                                     throw new RuntimeException(e);
 352                                 }
 353                             }))));
 354             futures.add(configFuture);
 355             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 356                 @Override
 357                 public void onSuccess(Void result) {
 358                     createFibEntries(identifier, update);
 359                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 360                             rd, update.getDestPrefix(), update.getRoutePaths());
 361                 }
 362
 363                 @Override
 364                 public void onFailure(Throwable throwable) {
 365                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 366                             update, throwable);
 367                 }
 368             }, MoreExecutors.directExecutor());
 369             return;
 370         }
 371
 372         //Handle all other routes only on a cluster reboot
 373         if (original.equals(update)) {
 374             //Reboot use-case
 375             createFibEntries(identifier, update);
 376             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 377                     rd, update.getDestPrefix(), update.getRoutePaths());
 378             return;
 379         }
 380
 381         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 382                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 383     }
 384
 385     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 386         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 387         List<SubTransaction> txnObjects =  new ArrayList<>();
 388         final VpnInstanceOpDataEntry vpnInstance =
 389                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 390         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 391         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 392                 + " has null vpnId!");
 393         final Collection<VpnToDpnList> vpnToDpnList;
 394         if (vrfEntry.getParentVpnRd() != null
 395                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 396             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 397             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 398             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 399                 vpnInstance.getVpnToDpnList();
 400             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 401                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 402         } else {
 403             vpnToDpnList = vpnInstance.getVpnToDpnList();
 404         }
 405         final Long vpnId = vpnInstance.getVpnId();
 406         final String rd = vrfTableKey.getRouteDistinguisher();
 407         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 408         if (subnetRoute != null) {
 409             final long elanTag = subnetRoute.getElantag();
 410             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 411                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 412             if (vpnToDpnList != null) {
 413                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 414                     () -> Collections.singletonList(
 415                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 416                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 417                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 418                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 419                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 420                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 421                                 }
 422                             }
 423                         })));
 424             }
 425             return;
 426         }
 427         // Get etherType value based on the IpPrefix address family type
 428         int etherType;
 429         try {
 430             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 431         } catch (IllegalArgumentException ex) {
 432             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 433             return;
 434         }
 435
 436         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 437         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 438             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 439                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 440                     final ReentrantLock lock = lockFor(vpnInstance);
 441                     lock.lock();
 442                     try {
 443                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 444                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 445                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 446                                     try {
 447                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 448                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 449                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 450                                                     TransactionAdapter.toWriteTransaction(tx),
 451                                                     txnObjects);
 452                                         } else {
 453                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 454                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 455                                         }
 456                                     } catch (NullPointerException e) {
 457                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 458                                                 vrfEntry.getDestPrefix(), e);
 459                                     }
 460                                 }
 461                             }
 462                         }
 463                     } finally {
 464                         lock.unlock();
 465                     }
 466                 })), MAX_RETRIES);
 467         }
 468
 469         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 470         if (optVpnUuid.isPresent()) {
 471             String vpnUuid = optVpnUuid.get();
 472             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 473             if (interVpnLink != null) {
 474                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 475                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 476                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 477                         // This is an static route that points to the other endpoint of an InterVpnLink
 478                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 479                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 480                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 481                     }
 482                 });
 483             }
 484         }
 485     }
 486
 487     void refreshFibTables(String rd, String prefix) {
 488         InstanceIdentifier<VrfEntry> vrfEntryId =
 489                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 490                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 491         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 492         if (vrfEntry.isPresent()) {
 493             createFibEntries(vrfEntryId, vrfEntry.get());
 494         }
 495     }
 496
 497     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 498         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 499         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 500         prefixBuilder.setDpnId(lri.getDpnId());
 501         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 502         prefixBuilder.setIpAddress(lri.getPrefix());
 503         // Increment the refCount here
 504         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 505             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 506         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 507         if (!isPresentInList) {
 508             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 509             List<String> vpnInstanceNames =
 510                 lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 511             vpnInstanceNames.add(vpnInstanceName);
 512             builder.setVpnInstanceList(vpnInstanceNames);
 513             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 514         } else {
 515             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 516         }
 517         return prefixBuilder.build();
 518     }
 519
 520     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 521             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 522         if (tx == null) {
 523             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 524                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 525                 "Error installing subnet route in FIB");
 526             return;
 527         }
 528         int etherType;
 529         try {
 530             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 531         } catch (IllegalArgumentException ex) {
 532             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 533             return;
 534         }
 535         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 536             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 537             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 538             final ReentrantLock lock = lockFor(lriKey);
 539             lock.lock();
 540             try {
 541                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 542                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 543
 544                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 545                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 546                                 fibUtil.getVpnInstanceOpData(rd);
 547                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 548                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 549                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 550                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 551                             }
 552                         }
 553                     }
 554                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 555                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 556                 }
 557             } finally {
 558                 lock.unlock();
 559             }
 560         });
 561         final List<InstructionInfo> instructions = new ArrayList<>();
 562         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 563             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 564         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 565         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 566         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 567                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 568         if (vrfEntry.getRoutePaths() != null) {
 569             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 570                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 571                     List<ActionInfo> actionsInfos = new ArrayList<>();
 572                     // reinitialize instructions list for LFIB Table
 573                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 574                     actionsInfos.add(new ActionPopMpls(etherType));
 575                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 576                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 577                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 578                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 579
 580                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 581                             NwConstants.ADD_FLOW, tx);
 582                 }
 583             }
 584         }
 585     }
 586
 587     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 588             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 589         List<MatchInfo> matches = new ArrayList<>();
 590
 591         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 592                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 593         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 594         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 595         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 596
 597         InetAddress destPrefix;
 598         try {
 599             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 600         } catch (UnknownHostException e) {
 601             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 602                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 603             return;
 604         }
 605
 606         // Match on VpnId and SubnetBroadCast IP address
 607         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 608         matches.add(MatchEthernetType.IPV4);
 609
 610         if (prefixLength != 0) {
 611             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 612         }
 613
 614         //Action is to drop the packet
 615         List<InstructionInfo> dropInstructions = new ArrayList<>();
 616         List<ActionInfo> actionsInfos = new ArrayList<>();
 617         actionsInfos.add(new ActionDrop());
 618         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 619
 620         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 621         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 622         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 623                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 624
 625         Flow flow = flowEntity.getFlowBuilder().build();
 626         String flowId = flowEntity.getFlowId();
 627         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 628         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 629
 630         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 631                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 632                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 633
 634         if (addOrRemove == NwConstants.ADD_FLOW) {
 635             tx.put(flowInstanceId,flow, true);
 636         } else {
 637             tx.delete(flowInstanceId);
 638         }
 639     }
 640
 641     /*
 642      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 643      * LportDispatcher table (via table 80)
 644      */
 645     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 646                                             final VrfEntry vrfEntry, int etherType) {
 647         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 648         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 649         // packet is commuted from Vpn2 to Vpn1.
 650         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 651         if (!interVpnLink.isActive()) {
 652             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 653                      interVpnLinkName, vrfEntry.getDestPrefix());
 654             return;
 655         }
 656
 657         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 658         if (!optLportTag.isPresent()) {
 659             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 660             return;
 661         }
 662
 663         Long lportTag = optLportTag.get();
 664         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 665         if (label == null) {
 666             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 667                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 668             return;
 669         }
 670         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 671         List<InstructionInfo> instructions = Arrays.asList(
 672             new InstructionApplyActions(actionsInfos),
 673             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 674                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 675                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 676                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 677             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 678         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 679         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 680
 681         for (BigInteger dpId : targetDpns) {
 682             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 683                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 684
 685             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 686                                /*writeTx*/null);
 687         }
 688     }
 689
 690
 691     /*
 692      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 693      */
 694     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 695                                                final VrfEntry vrfEntry, long vpnTag) {
 696         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 697         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 698             && vrfEntry.getRoutePaths().size() == 1);
 699         String destination = vrfEntry.getDestPrefix();
 700         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 701         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 702
 703         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 704         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 705         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 706         if (interVpnLink.getState().or(State.Error) != State.Active) {
 707             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 708                 destination, nextHop, interVpnLinkName);
 709             return;
 710         }
 711
 712         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 713         if (!optOtherEndpointLportTag.isPresent()) {
 714             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 715                 vpnUuid, interVpnLinkName);
 716             return;
 717         }
 718
 719         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 720         if (targetDpns.isEmpty()) {
 721             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 722                 vpnUuid, interVpnLinkName);
 723             return;
 724         }
 725
 726         String[] values = destination.split("/");
 727         String destPrefixIpAddress = values[0];
 728         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 729
 730         List<MatchInfo> matches = new ArrayList<>();
 731         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 732         matches.add(MatchEthernetType.IPV4);
 733
 734         if (prefixLength != 0) {
 735             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 736         }
 737
 738         List<Instruction> instructions =
 739             Arrays.asList(new InstructionWriteMetadata(
 740                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 741                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 742                             .L3VPN_SERVICE_INDEX)),
 743                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 744                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 745
 746         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 747         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 748         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 749             COOKIE_VM_FIB_TABLE, matches, instructions);
 750
 751         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 752             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 753
 754         for (BigInteger dpId : targetDpns) {
 755
 756             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 757                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 758                 dpId, interVpnLink.getInterVpnLinkName());
 759
 760             mdsalManager.installFlow(dpId, flowEntity);
 761         }
 762     }
 763
 764     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 765         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 766         String localNextHopIP = vrfEntry.getDestPrefix();
 767         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 768         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 769         if (localNextHopInfo == null) {
 770             boolean localNextHopSeen = false;
 771             List<Routes> vpnExtraRoutes = null;
 772             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 773             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 774             // FIXME: use an Identifier structure?
 775             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 776             lock.lock();
 777             try {
 778                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 779                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 780                         vpnName, usedRds, localNextHopIP);
 781                 if (LOG.isDebugEnabled()) {
 782                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 783                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 784                 }
 785
 786                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 787                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 788                     String ipPrefix;
 789                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 790                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 791                     } else {
 792                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 793                     }
 794                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 795                             ipPrefix);
 796                     if (localNextHopInfoLocal != null) {
 797                         localNextHopSeen = true;
 798                         BigInteger dpnId =
 799                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 800                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 801                         returnLocalDpnId.add(dpnId);
 802                     }
 803                 }
 804             } finally {
 805                 lock.unlock();
 806             }
 807             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 808                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 809                 if (optionalLabel.isPresent()) {
 810                     Long label = optionalLabel.get();
 811                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 812                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 813                     final ReentrantLock labelLock = lockFor(lriKey);
 814                     labelLock.lock();
 815                     try {
 816                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 817                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 818                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 819                                     fibUtil.getVpnInstanceOpData(rd);
 820                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 821                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 822                                 if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains(
 823                                        vpnInstanceName)) {
 824                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 825                                     localNextHopIP = lri.getPrefix();
 826                                 } else {
 827                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 828                                     localNextHopIP = lri.getPrefix();
 829                                 }
 830                             }
 831                             if (localNextHopInfo != null) {
 832                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 833                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 834                                 if (vpnExtraRoutes.isEmpty()) {
 835                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 836                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 837                                     returnLocalDpnId.add(dpnId);
 838                                 } else {
 839                                     for (Routes extraRoutes : vpnExtraRoutes) {
 840                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 841                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 842                                         returnLocalDpnId.add(dpnId);
 843                                     }
 844                                 }
 845                             }
 846                         }
 847                     } finally {
 848                         labelLock.unlock();
 849                     }
 850                 }
 851             }
 852             if (returnLocalDpnId.isEmpty()) {
 853                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 854             }
 855         } else {
 856             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 857                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 858             if (dpnId != null) {
 859                 returnLocalDpnId.add(dpnId);
 860             }
 861         }
 862         return returnLocalDpnId;
 863     }
 864
 865     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 866                                                 final Long vpnId, final String rd,
 867                                                 final VrfEntry vrfEntry,
 868                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 869                                                 int etherType) {
 870         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 871         if (localNextHopInfo != null) {
 872             long groupId;
 873             long localGroupId;
 874             final BigInteger dpnId = localNextHopInfo.getDpnId();
 875             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 876                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 877                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 878                 return dpnId;
 879             }
 880             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 881                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 882                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 883                 return dpnId;
 884             }
 885             if (!isVpnPresentInDpn(rd, dpnId)) {
 886                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 887                         vpnId, rd, dpnId.toString());
 888                 return BigInteger.ZERO;
 889             }
 890             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 891             String prefix = vrfEntry.getDestPrefix();
 892             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 893             //The loadbalancing group is created only if the extra route has multiple nexthops
 894             //to avoid loadbalancing the discovered routes
 895             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 896                     && routes != null) {
 897                 if (vpnExtraRoutes.size() > 1) {
 898                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 899                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 900                 } else {
 901                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 902                     localGroupId = groupId;
 903                 }
 904             } else {
 905                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 906                         gwMacAddress);
 907                 localGroupId = groupId;
 908             }
 909             if (groupId == FibConstants.INVALID_GROUP_ID) {
 910                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 911                         prefix, rd, interfaceName, dpnId.toString());
 912                 return BigInteger.ZERO;
 913             }
 914             final List<InstructionInfo> instructions = Collections.singletonList(
 915                     new InstructionApplyActions(
 916                             Collections.singletonList(new ActionGroup(groupId))));
 917             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 918                     new InstructionApplyActions(
 919                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 920             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 921             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 922             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 923             jobCoordinator.enqueueJob(jobKey,
 924                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 925                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 926                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 927                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 928                         optLabel.ifPresent(label -> {
 929                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 930                                 LOG.debug(
 931                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 932                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 933                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 934                                         nextHopAddressList);
 935                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 936                                         NwConstants.ADD_FLOW, tx);
 937                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 938                             } else {
 939                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 940                                                 + "route. LFib and Terminating table entries will not be created.",
 941                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 942                             }
 943                         });
 944                     }
 945                 })));
 946             return dpnId;
 947         }
 948         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 949                 vpnName);
 950         return BigInteger.ZERO;
 951     }
 952
 953     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 954         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 955         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 956         return dpnInVpn.isPresent();
 957     }
 958
 959     @Nullable
 960     private LabelRouteInfo getLabelRouteInfo(Long label) {
 961         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 962     }
 963
 964     @Nullable
 965     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 966         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 967             .child(LabelRouteInfo.class, label).build();
 968         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 969         if (opResult.isPresent()) {
 970             return opResult.get();
 971         }
 972         return null;
 973     }
 974
 975     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 976             @Nullable TypedWriteTransaction<Operational> tx) {
 977         if (lri == null) {
 978             return true;
 979         }
 980
 981         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 982         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 983             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 984
 985         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 986             ? lri.getVpnInstanceList() : new ArrayList<>();
 987         if (vpnInstancesList.contains(vpnInstanceName)) {
 988             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 989             vpnInstancesList.remove(vpnInstanceName);
 990         }
 991         if (vpnInstancesList.isEmpty()) {
 992             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 993             if (tx != null) {
 994                 tx.delete(lriId);
 995             } else {
 996                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 997             }
 998             return true;
 999         } else {
1000             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
1001             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
1002             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1003         }
1004         return false;
1005     }
1006
1007     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
1008                                       TypedWriteTransaction<Configuration> tx) {
1009         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1010
1011         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
1012
1013         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1014             dpId, label, groupId);
1015     }
1016
1017     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
1018                                                 TypedWriteTransaction<Configuration> tx) {
1019         List<MatchInfo> mkMatches = new ArrayList<>();
1020
1021         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1022             destDpId, label, actionsInfos);
1023
1024         // Matching metadata
1025         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1026         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1027
1028         List<InstructionInfo> mkInstructions = new ArrayList<>();
1029         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1030
1031         FlowEntity terminatingServiceTableFlowEntity =
1032             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1033             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1034                 String.format("%s:%d", "TST Flow Entry ", label),
1035             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1036
1037         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1038
1039         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1040
1041         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1042         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1043             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1044             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1045             .child(Flow.class, flowKey).build();
1046         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1047     }
1048
1049     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1050         FlowEntity flowEntity;
1051         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1052         List<MatchInfo> mkMatches = new ArrayList<>();
1053         // Matching metadata
1054         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1055         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1056             NwConstants.INTERNAL_TUNNEL_TABLE,
1057             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1058             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1059             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1060         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1061         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1062         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1063             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1064             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1065
1066         tx.delete(flowInstanceId);
1067         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1068     }
1069
1070     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1071         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1072         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1073         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1074         boolean shouldUpdateNonEcmpLocalNextHop = true;
1075         if (localNextHopInfo == null) {
1076             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1077             if (usedRds.size() > 1) {
1078                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1079                         vrfEntry.getDestPrefix(), vpnName, rd);
1080                 return returnLocalDpnId;
1081             }
1082             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1083             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1084             //in the vpn
1085             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1086                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1087             if (extraRouteOptional.isPresent()) {
1088                 Routes extraRoute = extraRouteOptional.get();
1089                 String ipPrefix;
1090                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1091                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1092                 } else {
1093                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1094                 }
1095                 if (extraRoute.getNexthopIpList().size() > 1) {
1096                     shouldUpdateNonEcmpLocalNextHop = false;
1097                 }
1098                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1099                 if (localNextHopInfo != null) {
1100                     String localNextHopIP = localNextHopInfo.getIpAddress();
1101                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1102                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1103                     if (!dpnId.equals(BigInteger.ZERO)) {
1104                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1105                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1106                         returnLocalDpnId.add(dpnId);
1107                     }
1108                 } else {
1109                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1110                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1111                 }
1112             }
1113
1114             if (localNextHopInfo == null) {
1115                 /* Imported VRF entry */
1116                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1117                 if (optionalLabel.isPresent()) {
1118                     Long label = optionalLabel.get();
1119                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1120                     LabelRouteInfo lri = getLabelRouteInfo(label);
1121                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1122                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1123                         prefixBuilder.setDpnId(lri.getDpnId());
1124                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1125                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1126                         if (!dpnId.equals(BigInteger.ZERO)) {
1127                             returnLocalDpnId.add(dpnId);
1128                         }
1129                     }
1130                 }
1131             }
1132
1133         } else {
1134             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1135             String localNextHopIP = localNextHopInfo.getIpAddress();
1136             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1137                     shouldUpdateNonEcmpLocalNextHop);
1138             if (!dpnId.equals(BigInteger.ZERO)) {
1139                 returnLocalDpnId.add(dpnId);
1140             }
1141         }
1142
1143         return returnLocalDpnId;
1144     }
1145
1146     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1147             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1148             boolean shouldUpdateNonEcmpLocalNextHop) {
1149         if (localNextHopInfo != null) {
1150             final BigInteger dpnId = localNextHopInfo.getDpnId();
1151             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1152                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1153                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1154                 return dpnId;
1155             }
1156             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1157                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1158                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1159                 return dpnId;
1160             }
1161
1162             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1163                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1164                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1165                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1166                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1167                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1168                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1169                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1170                                         NwConstants.DEL_FLOW, tx);
1171                                 removeTunnelTableEntry(dpnId, label, tx);
1172                             });
1173                         }
1174                     }
1175                 })));
1176             //TODO: verify below adjacency call need to be optimized (?)
1177             //In case of the removal of the extra route, the loadbalancing group is updated
1178             if (shouldUpdateNonEcmpLocalNextHop) {
1179                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1180             }
1181             return dpnId;
1182         }
1183         return BigInteger.ZERO;
1184     }
1185
1186     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1187             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1188         if (tx == null) {
1189             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1190                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1191                 "Error creating remote FIB entry");
1192             return;
1193         }
1194
1195         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1196         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", vrfEntry.getDestPrefix(), rd,
1197                 remoteDpnId);
1198
1199         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1200             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1201             return;
1202         }
1203         // Handling static VRF entries
1204         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1205         List<Routes> vpnExtraRoutes =
1206                 VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, vpnName, usedRds, vrfEntry.getDestPrefix());
1207         if (!vpnExtraRoutes.isEmpty()) {
1208             programRemoteFibWithLoadBalancingGroups(remoteDpnId, vpnId, rd, vrfEntry, vpnExtraRoutes);
1209         } else {
1210             // Program in case of other static VRF entries like floating IPs
1211             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1212         }
1213     }
1214
1215     private void programRemoteFibWithLoadBalancingGroups(final BigInteger remoteDpnId, final long vpnId, String rd,
1216             final VrfEntry vrfEntry, List<Routes> vpnExtraRoutes) {
1217         // create loadbalancing groups for extra routes only when the extra route is
1218         // present behind multiple VMs
1219         // Obtain the local routes for this particular dpn.
1220         java.util.Optional<Routes> routes = vpnExtraRoutes.stream().filter(route -> {
1221             Prefixes prefixToInterface =
1222                     fibUtil.getPrefixToInterface(vpnId, FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1223             if (prefixToInterface == null) {
1224                 return false;
1225             }
1226             return remoteDpnId.equals(prefixToInterface.getDpnId());
1227         }).findFirst();
1228         long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1229                 routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1230         if (groupId == FibConstants.INVALID_GROUP_ID) {
1231             LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", vrfEntry.getDestPrefix(), rd,
1232                     remoteDpnId);
1233             return;
1234         }
1235         List<ActionInfo> actionInfos = Collections.singletonList(new ActionGroup(groupId));
1236         List<InstructionInfo> instructions = Lists.newArrayList(new InstructionApplyActions(actionInfos));
1237         String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1238         jobCoordinator.enqueueJob(jobKey,
1239             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1240                 baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1241                         NwConstants.ADD_FLOW, txn, null);
1242             })));
1243
1244         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1245     }
1246
1247     private void programRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1248             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1249         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1250         if (adjacencyResults.isEmpty()) {
1251             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", vrfEntry.getRoutePaths(), rd,
1252                     remoteDpnId);
1253             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1254             return;
1255         }
1256         baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, TransactionAdapter.toWriteTransaction(tx),
1257                 rd, adjacencyResults, null);
1258         LOG.debug("Successfully programmed FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1259     }
1260
1261     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1262     /* Get interface info from prefix to interface mapping;
1263         Use the interface info to get the corresponding vpn interface op DS entry,
1264         remove the adjacency corresponding to this fib entry.
1265         If adjacency removed is the last adjacency, clean up the following:
1266          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1267          - prefix to interface entry
1268          - vpn interface op DS
1269      */
1270         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1271         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1272         if (prefixInfo == null) {
1273             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1274             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1275             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1276             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
1277                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1278                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1279                     if (nextHopIp != null) {
1280                         String ipPrefix;
1281                         if (isIpv4Address(nextHopIp)) {
1282                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1283                         } else {
1284                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1285                         }
1286                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1287                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1288                     }
1289                 }
1290             }
1291             if (prefixInfo == null) {
1292                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1293                 if (optionalLabel.isPresent()) {
1294                     Long label = optionalLabel.get();
1295                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1296                     LabelRouteInfo lri = getLabelRouteInfo(label);
1297                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1298                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1299                         prefixBuilder.setDpnId(lri.getDpnId());
1300                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1301                         prefixBuilder.setIpAddress(lri.getPrefix());
1302                         prefixInfo = prefixBuilder.build();
1303                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1304                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1305                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1306                     }
1307                 }
1308             }
1309         } else {
1310             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1311         }
1312     }
1313
1314     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1315                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1316
1317         if (prefixInfo == null) {
1318             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1319                     vrfEntry.getDestPrefix(), vpnId, rd);
1320             return; //Don't have any info for this prefix (shouldn't happen); need to return
1321         }
1322
1323         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1324             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1325                     vrfEntry.getDestPrefix(), vpnId, rd);
1326             return;
1327         }
1328
1329         String ifName = prefixInfo.getVpnInterfaceName();
1330         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1331             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1332     }
1333
1334     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1335         Prefixes prefixInfo;
1336         Long vpnId;
1337         String rd;
1338         VrfEntry vrfEntry;
1339         Routes extraRoute;
1340
1341         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1342                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1343             this.prefixInfo = prefixInfo;
1344             this.vpnId = vpnId;
1345             this.rd = rd;
1346             this.vrfEntry = vrfEntry;
1347             this.extraRoute = extraRoute;
1348         }
1349
1350         @Override
1351         public List<ListenableFuture<Void>> call() {
1352             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1353             // to call the respective helpers.
1354             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1355                 //First Cleanup LabelRouteInfo
1356                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1357                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1358                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1359                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1360                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1361                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1362                         final ReentrantLock lock = lockFor(lriKey);
1363                         lock.lock();
1364                         try {
1365                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1366                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1367                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1368                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1369                                         fibUtil.getVpnInstanceOpData(rd);
1370                                 String vpnInstanceName = "";
1371                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1372                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1373                                 }
1374                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1375                                 if (lriRemoved) {
1376                                     String parentRd = lri.getParentVpnRd();
1377                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1378                                             parentRd, vrfEntry.getDestPrefix()));
1379                                 }
1380                             } else {
1381                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1382                                         rd, vrfEntry.getDestPrefix()));
1383                             }
1384                         } finally {
1385                             lock.unlock();
1386                         }
1387                     });
1388                 }
1389                 String ifName = prefixInfo.getVpnInterfaceName();
1390                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1391                 String vpnName = null;
1392
1393                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1394                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1395                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1396                     if (vpnNameOpt.isPresent()) {
1397                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1398                     }
1399                 }
1400                 if (optVpnName.isPresent()) {
1401                     vpnName = optVpnName.get();
1402                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1403                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1404                     if (opVpnInterface.isPresent()) {
1405                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1406                         if (vpnId != associatedVpnId) {
1407                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1408                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1409                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1410                             return;
1411                         } else {
1412                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1413                                     vrfEntry.getDestPrefix(), associatedVpnId);
1414                         }
1415                     }
1416                 }
1417                 if (extraRoute != null) {
1418                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1419                     //Only one used Rd present in case of removal event
1420                     String usedRd = usedRds.get(0);
1421                     if (optVpnName.isPresent()) {
1422                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1423                                         vrfEntry.getDestPrefix()));
1424                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1425                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1426                     }
1427                 }
1428                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1429             }));
1430         }
1431     }
1432
1433     /**
1434      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1435      * Remove Adjacency from VPNInterfaceOpData.
1436      * if Adjacency != primary.
1437      * if Adjacency == primary , then mark it for deletion.
1438      * Remove entire VPNinterfaceOpData Entry.
1439      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1440      * @param vrfEntry - VrfEntry removed
1441      * @param ifName - Interface name from VRFentry
1442      * @param vpnName - VPN name of corresponding VRF
1443      * @param tx - ReadWrite Tx
1444      */
1445     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1446                                                           TypedReadWriteTransaction<Operational> tx)
1447             throws ExecutionException, InterruptedException {
1448         InstanceIdentifier<Adjacency> adjacencyIid =
1449                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1450         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1451         if (adjacencyOptional.isPresent()) {
1452             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1453                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1454             } else {
1455                 tx.merge(adjacencyIid,
1456                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1457             }
1458         }
1459
1460         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1461
1462         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1463             return;
1464         }
1465
1466         @NonNull List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
1467         if (adjacencies.size() <= 2
1468                 && adjacencies.stream().allMatch(adjacency ->
1469                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1470                         && adjacency.isMarkedForDeletion() != null
1471                         && adjacency.isMarkedForDeletion()
1472         )) {
1473             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1474             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1475         }
1476     }
1477
1478     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1479         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1480         final String rd = vrfTableKey.getRouteDistinguisher();
1481         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1482         if (vpnInstance == null) {
1483             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1484             return;
1485         }
1486         final Collection<VpnToDpnList> vpnToDpnList;
1487         if (vrfEntry.getParentVpnRd() != null
1488                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1489             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1490             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1491             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1492                     vpnInstance.getVpnToDpnList();
1493             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1494                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1495         } else {
1496             vpnToDpnList = vpnInstance.getVpnToDpnList();
1497         }
1498
1499         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1500         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1501         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1502         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1503         if (subnetRoute != null) {
1504             long elanTag = subnetRoute.getElantag();
1505             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1506                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1507             if (vpnToDpnList != null) {
1508                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1509                     () -> Collections.singletonList(
1510                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1511                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1512
1513                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1514                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1515                                     TransactionAdapter.toWriteTransaction(tx), null);
1516                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1517                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1518                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1519                                 }
1520
1521                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1522                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1523                             }
1524                         })));
1525             }
1526             optionalLabel.ifPresent(label -> {
1527                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1528                 final ReentrantLock lock = lockFor(lriKey);
1529                 lock.lock();
1530                 try {
1531                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1532                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1533                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1534                                 fibUtil.getVpnInstanceOpData(rd);
1535                         String vpnInstanceName = "";
1536                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1537                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1538                         }
1539                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1540                         if (lriRemoved) {
1541                             String parentRd = lri.getParentVpnRd();
1542                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1543                                     parentRd, vrfEntry.getDestPrefix()));
1544                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1545                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1546                         }
1547                     } else {
1548                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1549                                 rd, vrfEntry.getDestPrefix()));
1550                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1551                                 label, rd, vrfEntry.getDestPrefix());
1552                     }
1553                 } finally {
1554                     lock.unlock();
1555                 }
1556             });
1557             return;
1558         }
1559
1560         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1561             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1562         if (vpnToDpnList != null) {
1563             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1564                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1565             String jobKey;
1566             Optional<Routes> extraRouteOptional;
1567             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1568             if (usedRds != null && !usedRds.isEmpty()) {
1569                 if (usedRds.size() > 1) {
1570                     LOG.error("The extra route prefix is still present in some DPNs");
1571                     return ;
1572                 } else {
1573                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1574                     //is not present in any other DPN
1575                     extraRouteOptional = VpnExtraRouteHelper
1576                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1577                 }
1578             } else {
1579                 extraRouteOptional = Optional.absent();
1580             }
1581
1582             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1583                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1584                     if (localDpnIdList.size() <= 0) {
1585                         for (VpnToDpnList curDpn : vpnToDpnList) {
1586                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1587                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1588                                 TransactionAdapter.toWriteTransaction(tx));
1589                         }
1590                     } else {
1591                         for (BigInteger localDpnId : localDpnIdList) {
1592                             for (VpnToDpnList curDpn : vpnToDpnList) {
1593                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1594                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1595                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1596                                         TransactionAdapter.toWriteTransaction(tx));
1597                                 }
1598                             }
1599                         }
1600                     }
1601                     if (extraRouteOptional.isPresent()) {
1602                         //Remove select groups only for extra-routes
1603                         nextHopManager.removeNextHopPointer(nextHopManager
1604                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1605                         nextHopManager.removeNextHopPointer(nextHopManager
1606                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1607                     }
1608                 })), MAX_RETRIES);
1609         }
1610
1611         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1612         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1613         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1614
1615         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1616         // of the interVpnLink.
1617         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1618         if (optVpnUuid.isPresent()) {
1619             String vpnUuid = optVpnUuid.get();
1620             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1621                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1622                 if (optInterVpnLink.isPresent()) {
1623                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1624                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1625                         // This is route that points to the other endpoint of an InterVpnLink
1626                         // In that case, we should look for the FIB table pointing to
1627                         // LPortDispatcher table and remove it.
1628                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1629                     }
1630                 }
1631             });
1632         }
1633
1634     }
1635
1636     private void makeLFibTableEntry(BigInteger dpId, long label, @Nullable List<InstructionInfo> instructions,
1637                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1638         if (tx == null) {
1639             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1640                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1641                 "Error making LFIB table entry");
1642             return;
1643         }
1644
1645         List<MatchInfo> matches = new ArrayList<>();
1646         matches.add(MatchEthernetType.MPLS_UNICAST);
1647         matches.add(new MatchMplsLabel(label));
1648
1649         // Install the flow entry in L3_LFIB_TABLE
1650         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1651
1652         FlowEntity flowEntity;
1653         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1654             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1655         Flow flow = flowEntity.getFlowBuilder().build();
1656         String flowId = flowEntity.getFlowId();
1657         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1658         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1659         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1660             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1661             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1662
1663         if (addOrRemove == NwConstants.ADD_FLOW) {
1664             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1665         } else {
1666             tx.delete(flowInstanceId);
1667         }
1668
1669         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1670             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1671     }
1672
1673     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1674                                     final FutureCallback<List<Void>> callback) {
1675         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1676         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1677             () -> {
1678                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1679                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1680                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1681                         LogicalDatastoreType.CONFIGURATION, id);
1682                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1683                 if (!vrfTable.isPresent()) {
1684                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1685                     if (callback != null) {
1686                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1687                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1688                     }
1689                     return futures;
1690                 }
1691
1692                 final ReentrantLock lock = lockFor(vpnInstance);
1693                 lock.lock();
1694                 try {
1695                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1696                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1697                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1698                             if (subnetRoute != null) {
1699                                 long elanTag = subnetRoute.getElantag();
1700                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1701                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1702                                         tx);
1703                                 continue;
1704                             }
1705                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1706                             if (routerInt != null) {
1707                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1708                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1709                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1710                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1711                                         NwConstants.ADD_FLOW);
1712                                 continue;
1713                             }
1714                             //Handle local flow creation for imports
1715                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1716                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1717                                 if (optionalLabel.isPresent()) {
1718                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1719                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1720                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1721                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1722                                             try {
1723                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1724                                                         vrfEntry.getDestPrefix());
1725                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1726                                             } catch (IllegalArgumentException ex) {
1727                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1728                                                         vrfEntry.getDestPrefix());
1729                                             }
1730                                             continue;
1731                                         }
1732                                     }
1733                                 }
1734                             }
1735                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1736                                     vrfEntry, dpnId);
1737                             if (shouldCreateRemoteFibEntry) {
1738                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1739                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1740                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1741                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1742                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1743                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1744                                 } else {
1745                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1746                                             vrfEntry, tx);
1747                                 }
1748                             }
1749                         }
1750                     }));
1751                     if (callback != null) {
1752                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1753                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1754                     }
1755                 } finally {
1756                     lock.unlock();
1757                 }
1758                 return futures;
1759             });
1760     }
1761
1762     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1763                                             final String localNextHopIp, final String remoteNextHopIp) {
1764         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1765             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1766         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1767         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1768         List<SubTransaction> txnObjects =  new ArrayList<>();
1769         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1770         if (vrfTable.isPresent()) {
1771             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1772                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1773                     final ReentrantLock lock = lockFor(vpnInstance);
1774                     lock.lock();
1775                     try {
1776                         vrfTable.get().nonnullVrfEntry().stream()
1777                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1778                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1779                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1780                     } finally {
1781                         lock.unlock();
1782                     }
1783                 })));
1784         }
1785     }
1786
1787     public void manageRemoteRouteOnDPN(final boolean action,
1788                                        final BigInteger localDpnId,
1789                                        final long vpnId,
1790                                        final String rd,
1791                                        final String destPrefix,
1792                                        final String destTepIp,
1793                                        final long label) {
1794         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1795
1796         if (vpnInstance == null) {
1797             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1798             return;
1799         }
1800
1801         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1802             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1803                 final ReentrantLock lock = lockFor(vpnInstance);
1804                 lock.lock();
1805                 try {
1806                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1807                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1808                     if (vrfEntry == null) {
1809                         return;
1810                     }
1811                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1812                             action, localDpnId, vpnId, rd, destPrefix);
1813                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1814                     VrfEntry modVrfEntry;
1815                     if (routePathList == null || routePathList.isEmpty()) {
1816                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1817                                 Collections.singletonList(destTepIp),
1818                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1819                     } else {
1820                         modVrfEntry = vrfEntry;
1821                     }
1822
1823                     if (action) {
1824                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1825                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1826                                 modVrfEntry, tx);
1827                     } else {
1828                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1829                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1830                                 vrfEntry.getDestPrefix());
1831                         if (usedRds.size() > 1) {
1832                             LOG.debug("The extra route prefix is still present in some DPNs");
1833                             return;
1834                         }
1835                         //Is this fib route an extra route? If yes, get the nexthop which would be
1836                         //an adjacency in the vpn
1837                         Optional<Routes> extraRouteOptional = Optional.absent();
1838                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC && usedRds.size() != 0) {
1839                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1840                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1841                                     usedRds.get(0), vrfEntry.getDestPrefix());
1842                         }
1843                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1844                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1845                     }
1846                 } finally {
1847                     lock.unlock();
1848                 }
1849             })));
1850     }
1851
1852     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1853                                  final FutureCallback<List<Void>> callback) {
1854         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1855         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1856             () -> {
1857                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1858                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1859                 List<SubTransaction> txnObjects =  new ArrayList<>();
1860                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1861                         LogicalDatastoreType.CONFIGURATION, id);
1862                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1863                 if (vrfTable.isPresent()) {
1864                     final ReentrantLock lock = lockFor(vpnInstance);
1865                     lock.lock();
1866                     try {
1867                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1868                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1869                             for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1870                                 /* parentRd is only filled for external PNF cases where the interface on the external
1871                                  * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1872                                  * #fibUtil.isInterfacePresentInDpn().
1873                                 * */
1874                                 String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1875                                         : rd;
1876                                 /* Handle subnet routes here */
1877                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1878                                 if (subnetRoute != null && !fibUtil
1879                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1880                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1881                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1882                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1883                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1884                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1885                                     if (routePaths != null) {
1886                                         for (RoutePaths routePath : routePaths) {
1887                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1888                                                     DEFAULT_FIB_FLOW_PRIORITY,
1889                                                     NwConstants.DEL_FLOW, tx);
1890                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1891                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1892                                                     vrfEntry.getDestPrefix());
1893                                         }
1894                                     }
1895                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1896                                             NwConstants.DEL_FLOW, tx);
1897                                     continue;
1898                                 }
1899                                 // ping responder for router interfaces
1900                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1901                                 if (routerInt != null) {
1902                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1903                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1904                                             routerInt.getMacAddress());
1905                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1906                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1907                                             NwConstants.DEL_FLOW);
1908                                     continue;
1909                                 }
1910
1911                                 //Handle local flow deletion for imports
1912                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1913                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1914                                     if (optionalLabel.isPresent()) {
1915                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1916                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1917                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1918                                                 lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1919                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1920                                         }
1921                                     }
1922                                 }
1923
1924                                 // Passing null as we don't know the dpn
1925                                 // to which prefix is attached at this point
1926                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1927                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1928                                 Optional<Routes> extraRouteOptional;
1929                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1930                                 //an adjacency in the vpn
1931                                 if (usedRds != null && !usedRds.isEmpty()) {
1932                                     if (usedRds.size() > 1) {
1933                                         LOG.error("The extra route prefix is still present in some DPNs");
1934                                         return;
1935                                     } else {
1936                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1937                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1938
1939                                     }
1940                                 } else {
1941                                     extraRouteOptional = Optional.absent();
1942                                 }
1943                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1944                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1945                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1946                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1947                                 } else {
1948                                     if (subnetRoute == null || !fibUtil
1949                                             .isInterfacePresentInDpn(parentRd, dpnId)) {
1950                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1951                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1952                                             TransactionAdapter.toWriteTransaction(tx));
1953                                     }
1954                                 }
1955                             }
1956                         }));
1957                     } finally {
1958                         lock.unlock();
1959                     }
1960                     if (callback != null) {
1961                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1962                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1963                     }
1964                 } else {
1965                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1966                 }
1967                 return futures;
1968             });
1969
1970     }
1971
1972     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1973                                            final String localNextHopIp, final String remoteNextHopIp) {
1974         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1975                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1976             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1977         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1978         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1979         List<SubTransaction> txnObjects =  new ArrayList<>();
1980         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1981         if (vrfTable.isPresent()) {
1982             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1983                 () -> {
1984                     final ReentrantLock lock = lockFor(vpnInstance);
1985                     lock.lock();
1986                     try {
1987                         return Collections.singletonList(
1988                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1989                                 tx -> vrfTable.get().nonnullVrfEntry().stream()
1990                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1991                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1992                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1993                                         txnObjects))));
1994                     } finally {
1995                         lock.unlock();
1996                     }
1997                 });
1998         }
1999     }
2000
2001     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2002         InstanceIdentifierBuilder<VrfTables> idBuilder =
2003             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2004         return idBuilder.build();
2005     }
2006
2007     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2008         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2009                 .FLOWID_SEPARATOR + nextHop;
2010     }
2011
2012     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2013         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2014                 + tableMiss + FLOWID_PREFIX;
2015     }
2016
2017     @Nullable
2018     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2019         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2020             .child(VrfTables.class, new VrfTablesKey(rd))
2021             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2022         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2023         if (vrfEntry.isPresent()) {
2024             return vrfEntry.get();
2025         }
2026         return null;
2027     }
2028
2029     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2030                                              final String vpnName,
2031                                              final VrfEntry vrfEntry) {
2032         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2033
2034         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2035         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2036
2037         if (targetDpns.isEmpty()) {
2038             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2039             return;
2040         }
2041
2042         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2043         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2044
2045         // delete from FIB
2046         //
2047         optNextHop.ifPresent(nextHop -> {
2048             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2049             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2050             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2051                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2052
2053             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2054             for (BigInteger dpId : targetDpns) {
2055                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2056                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2057
2058                 mdsalManager.removeFlow(dpId, flow);
2059             }
2060         });
2061
2062         // delete from LFIB
2063         //
2064         optLabel.ifPresent(label -> {
2065             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2066
2067             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2068                 for (BigInteger dpId : targetDpns) {
2069                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2070                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2071                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
2072                             tx);
2073                 }
2074             }), LOG, "Error removing flows");
2075         });
2076     }
2077
2078     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2079             List<String> nextHopAddressList, LabelRouteInfo lri) {
2080         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2081                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2082     }
2083
2084     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2085         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2086             return true;
2087         }
2088
2089         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2090         if (prefix != null) {
2091             BigInteger prefixDpnId = prefix.getDpnId();
2092             if (dpnId.equals(prefixDpnId)) {
2093                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2094                         vrfEntry, dpnId);
2095                 return false;
2096             }
2097         }
2098         return true;
2099     }
2100
2101     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2102         // FIXME: use vpnInstance.key() instead?
2103         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2104     }
2105
2106     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2107         return JvmGlobalLocks.getLockFor(label);
2108     }
2109 }