fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14 import static org.opendaylight.netvirt.fibmanager.FibUtil.nullToEmpty;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.net.InetAddress;
  25 import java.net.UnknownHostException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.List;
  31 import java.util.Objects;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.locks.ReentrantLock;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.inject.Inject;
  39 import javax.inject.Singleton;
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.Datastore.Operational;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  48 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  49 import org.opendaylight.genius.infra.TransactionAdapter;
  50 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  51 import org.opendaylight.genius.infra.TypedWriteTransaction;
  52 import org.opendaylight.genius.mdsalutil.ActionInfo;
  53 import org.opendaylight.genius.mdsalutil.FlowEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NWUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  66 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  72 import org.opendaylight.genius.utils.JvmGlobalLocks;
  73 import org.opendaylight.genius.utils.ServiceIndex;
  74 import org.opendaylight.genius.utils.batching.SubTransaction;
  75 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  76 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  77 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  78 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  79 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  85 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124
 125 @Singleton
 126 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 127
 128     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 129     private static final String FLOWID_PREFIX = "L3.";
 130     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 131     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 132     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 133     private static final int LFIB_INTERVPN_PRIORITY = 15;
 134     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 135     private static final int MAX_RETRIES = 3;
 136     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 137
 138     private final DataBroker dataBroker;
 139     private final ManagedNewTransactionRunner txRunner;
 140     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 141     private final IMdsalApiManager mdsalManager;
 142     private final NexthopManager nextHopManager;
 143     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 144     private final BaseVrfEntryHandler baseVrfEntryHandler;
 145     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 146     private final JobCoordinator jobCoordinator;
 147     private final IElanService elanManager;
 148     private final FibUtil fibUtil;
 149     private final InterVpnLinkCache interVpnLinkCache;
 150     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 151     private final UpgradeState upgradeState;
 152     private final DataTreeEventCallbackRegistrar eventCallbacks;
 153
 154     @Inject
 155     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 156                             final NexthopManager nexthopManager,
 157                             final IElanService elanManager,
 158                             final BaseVrfEntryHandler vrfEntryHandler,
 159                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 160                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 161                             final JobCoordinator jobCoordinator,
 162                             final FibUtil fibUtil,
 163                             final InterVpnLinkCache interVpnLinkCache,
 164                             final UpgradeState upgradeState,
 165                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 166         super(VrfEntry.class, VrfEntryListener.class);
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 170         this.mdsalManager = mdsalApiManager;
 171         this.nextHopManager = nexthopManager;
 172         this.elanManager = elanManager;
 173         this.baseVrfEntryHandler = vrfEntryHandler;
 174         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 175         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 176         this.jobCoordinator = jobCoordinator;
 177         this.fibUtil = fibUtil;
 178         this.interVpnLinkCache = interVpnLinkCache;
 179         this.upgradeState = upgradeState;
 180         this.eventCallbacks = eventCallbacks;
 181     }
 182
 183     @Override
 184     @PostConstruct
 185     public void init() {
 186         LOG.info("{} init", getClass().getSimpleName());
 187         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 188     }
 189
 190     @Override
 191     @SuppressWarnings("checkstyle:IllegalCatch")
 192     public void close() {
 193         closeables.forEach(c -> {
 194             try {
 195                 c.close();
 196             } catch (Exception e) {
 197                 LOG.warn("Error closing {}", c, e);
 198             }
 199         });
 200     }
 201
 202     @Override
 203     protected VrfEntryListener getDataTreeChangeListener() {
 204         return VrfEntryListener.this;
 205     }
 206
 207     @Override
 208     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 209         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 210     }
 211
 212     @Override
 213     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 214         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 215         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 216         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 217                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 218         addFibEntries(identifier, vrfEntry, rd);
 219         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 220                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 221     }
 222
 223     //This method is temporary. Eventually Factory design pattern will be used to get
 224     // right VrfEntryhandle and invoke its methods.
 225     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 226         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 227             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 228             return;
 229         }
 230         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 231             LOG.info("EVPN flows need to be programmed.");
 232             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 233                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 234             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 235             closeables.add(evpnVrfEntryHandler);
 236             return;
 237         }
 238         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 239         if (routerInt != null) {
 240             // ping responder for router interfaces
 241             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 242             return;
 243         }
 244         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 245             createFibEntries(identifier, vrfEntry);
 246             return;
 247         }
 248     }
 249
 250     @Override
 251     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 252         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 253         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 254         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 255                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 256         removeFibEntries(identifier, vrfEntry, rd);
 257         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 258             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 259     }
 260
 261     //This method is temporary. Eventually Factory design pattern will be used to get
 262     // right VrfEntryhandle and invoke its methods.
 263     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 264         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 265             LOG.info("EVPN flows to be deleted");
 266             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 267                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 268             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 269             closeables.add(evpnVrfEntryHandler);
 270             return;
 271         }
 272         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 273         if (routerInt != null) {
 274             // ping responder for router interfaces
 275             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 276             return;
 277         }
 278         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 279             deleteFibEntries(identifier, vrfEntry);
 280             return;
 281         }
 282         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 283             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 284             return;
 285         }
 286     }
 287
 288     @Override
 289     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 290     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 291     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 292     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 293         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 294         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 295         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 296                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 297         // Handle BGP Routes first
 298         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 299             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 300             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 301                     rd, update.getDestPrefix(), update.getRoutePaths());
 302             return;
 303         }
 304
 305         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 306             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 307             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 308             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 309
 310             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 311             //updated vrf entry route path is null. This can happen during tunnel events.
 312             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 313             List<String> usedRds = new ArrayList<>();
 314             if (optVpnInstance.isPresent()) {
 315                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 316                         update.getDestPrefix());
 317             }
 318             // If original VRF Entry had nexthop null , but update VRF Entry
 319             // has nexthop , route needs to be created on remote Dpns
 320             if (originalRoutePath == null || originalRoutePath.isEmpty()
 321                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 322                 // TODO(vivek): Though ugly, Not handling this code now, as each
 323                 // tep add event will invoke flow addition
 324                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 325                         + " This event is IGNORED here.", update.getDestPrefix());
 326                 return;
 327             }
 328
 329             // If original VRF Entry had valid nexthop , but update VRF Entry
 330             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 331             if (updateRoutePath == null || updateRoutePath.isEmpty()
 332                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 333                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 334                         + "This event is IGNORED here.", update.getDestPrefix());
 335                 return;
 336             }
 337             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 338             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 339             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 340             List<ListenableFuture<Void>> futures = new ArrayList<>();
 341             ListenableFuture<Void> configFuture =
 342                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 343                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 344                         nextHopsRemoved.parallelStream()
 345                             .forEach(nextHopRemoved -> {
 346                                 try {
 347                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 348                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 349                                 } catch (ExecutionException | InterruptedException e) {
 350                                     throw new RuntimeException(e);
 351                                 }
 352                             }))));
 353             futures.add(configFuture);
 354             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 355                 @Override
 356                 public void onSuccess(Void result) {
 357                     createFibEntries(identifier, update);
 358                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 359                             rd, update.getDestPrefix(), update.getRoutePaths());
 360                 }
 361
 362                 @Override
 363                 public void onFailure(Throwable throwable) {
 364                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 365                             update, throwable);
 366                 }
 367             }, MoreExecutors.directExecutor());
 368             return;
 369         }
 370
 371         //Handle all other routes only on a cluster reboot
 372         if (original.equals(update)) {
 373             //Reboot use-case
 374             createFibEntries(identifier, update);
 375             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 376                     rd, update.getDestPrefix(), update.getRoutePaths());
 377             return;
 378         }
 379
 380         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 381                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 382     }
 383
 384     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 385         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 386         List<SubTransaction> txnObjects =  new ArrayList<>();
 387         final VpnInstanceOpDataEntry vpnInstance =
 388                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 389         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 390         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 391                 + " has null vpnId!");
 392         final Collection<VpnToDpnList> vpnToDpnList;
 393         if (vrfEntry.getParentVpnRd() != null
 394                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 395             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 396             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 397             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 398                 vpnInstance.getVpnToDpnList();
 399             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 400                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 401         } else {
 402             vpnToDpnList = vpnInstance.getVpnToDpnList();
 403         }
 404         final Long vpnId = vpnInstance.getVpnId();
 405         final String rd = vrfTableKey.getRouteDistinguisher();
 406         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 407         if (subnetRoute != null) {
 408             final long elanTag = subnetRoute.getElantag();
 409             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 410                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 411             if (vpnToDpnList != null) {
 412                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 413                     () -> Collections.singletonList(
 414                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 415                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 416                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 417                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 418                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 419                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 420                                 }
 421                             }
 422                         })));
 423             }
 424             return;
 425         }
 426         // Get etherType value based on the IpPrefix address family type
 427         int etherType;
 428         try {
 429             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 430         } catch (IllegalArgumentException ex) {
 431             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 432             return;
 433         }
 434
 435         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 436         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 437             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 438                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 439                     final ReentrantLock lock = lockFor(vpnInstance);
 440                     lock.lock();
 441                     try {
 442                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 443                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 444                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 445                                     try {
 446                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 447                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 448                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 449                                                     TransactionAdapter.toWriteTransaction(tx),
 450                                                     txnObjects);
 451                                         } else {
 452                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 453                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 454                                         }
 455                                     } catch (NullPointerException e) {
 456                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 457                                                 vrfEntry.getDestPrefix(), e);
 458                                     }
 459                                 }
 460                             }
 461                         }
 462                     } finally {
 463                         lock.unlock();
 464                     }
 465                 })), MAX_RETRIES);
 466         }
 467
 468         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 469         if (optVpnUuid.isPresent()) {
 470             String vpnUuid = optVpnUuid.get();
 471             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 472             if (interVpnLink != null) {
 473                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 474                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 475                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 476                         // This is an static route that points to the other endpoint of an InterVpnLink
 477                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 478                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 479                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 480                     }
 481                 });
 482             }
 483         }
 484     }
 485
 486     void refreshFibTables(String rd, String prefix) {
 487         InstanceIdentifier<VrfEntry> vrfEntryId =
 488                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 489                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 490         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 491         if (vrfEntry.isPresent()) {
 492             createFibEntries(vrfEntryId, vrfEntry.get());
 493         }
 494     }
 495
 496     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 497         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 498         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 499         prefixBuilder.setDpnId(lri.getDpnId());
 500         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 501         prefixBuilder.setIpAddress(lri.getPrefix());
 502         // Increment the refCount here
 503         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 504             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 505         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 506         if (!isPresentInList) {
 507             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 508             List<String> vpnInstanceNames = new ArrayList<>(nullToEmpty(lri.getVpnInstanceList()));
 509             vpnInstanceNames.add(vpnInstanceName);
 510             builder.setVpnInstanceList(vpnInstanceNames);
 511             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 512         } else {
 513             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 514         }
 515         return prefixBuilder.build();
 516     }
 517
 518     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 519             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 520         if (tx == null) {
 521             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 522                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 523                 "Error installing subnet route in FIB");
 524             return;
 525         }
 526         int etherType;
 527         try {
 528             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 529         } catch (IllegalArgumentException ex) {
 530             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 531             return;
 532         }
 533         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 534             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 535             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 536             final ReentrantLock lock = lockFor(lriKey);
 537             lock.lock();
 538             try {
 539                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 540                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 541
 542                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 543                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 544                                 fibUtil.getVpnInstanceOpData(rd);
 545                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 546                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 547                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 548                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 549                             }
 550                         }
 551                     }
 552                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 553                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 554                 }
 555             } finally {
 556                 lock.unlock();
 557             }
 558         });
 559         final List<InstructionInfo> instructions = new ArrayList<>();
 560         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 561             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 562         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 563         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 564         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 565                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 566         if (vrfEntry.getRoutePaths() != null) {
 567             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 568                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 569                     List<ActionInfo> actionsInfos = new ArrayList<>();
 570                     // reinitialize instructions list for LFIB Table
 571                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 572                     actionsInfos.add(new ActionPopMpls(etherType));
 573                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 574                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 575                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 576                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 577
 578                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 579                             NwConstants.ADD_FLOW, tx);
 580                 }
 581             }
 582         }
 583     }
 584
 585     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 586             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 587         List<MatchInfo> matches = new ArrayList<>();
 588
 589         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 590                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 591         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 592         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 593         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 594
 595         InetAddress destPrefix;
 596         try {
 597             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 598         } catch (UnknownHostException e) {
 599             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 600                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 601             return;
 602         }
 603
 604         // Match on VpnId and SubnetBroadCast IP address
 605         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 606         matches.add(MatchEthernetType.IPV4);
 607
 608         if (prefixLength != 0) {
 609             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 610         }
 611
 612         //Action is to drop the packet
 613         List<InstructionInfo> dropInstructions = new ArrayList<>();
 614         List<ActionInfo> actionsInfos = new ArrayList<>();
 615         actionsInfos.add(new ActionDrop());
 616         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 617
 618         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 619         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 620         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 621                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 622
 623         Flow flow = flowEntity.getFlowBuilder().build();
 624         String flowId = flowEntity.getFlowId();
 625         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 626         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 627
 628         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 629                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 630                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 631
 632         if (addOrRemove == NwConstants.ADD_FLOW) {
 633             tx.put(flowInstanceId,flow, true);
 634         } else {
 635             tx.delete(flowInstanceId);
 636         }
 637     }
 638
 639     /*
 640      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 641      * LportDispatcher table (via table 80)
 642      */
 643     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 644                                             final VrfEntry vrfEntry, int etherType) {
 645         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 646         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 647         // packet is commuted from Vpn2 to Vpn1.
 648         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 649         if (!interVpnLink.isActive()) {
 650             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 651                      interVpnLinkName, vrfEntry.getDestPrefix());
 652             return;
 653         }
 654
 655         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 656         if (!optLportTag.isPresent()) {
 657             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 658             return;
 659         }
 660
 661         Long lportTag = optLportTag.get();
 662         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 663         if (label == null) {
 664             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 665                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 666             return;
 667         }
 668         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 669         List<InstructionInfo> instructions = Arrays.asList(
 670             new InstructionApplyActions(actionsInfos),
 671             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 672                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 673                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 674                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 675             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 676         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 677         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 678
 679         for (BigInteger dpId : targetDpns) {
 680             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 681                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 682
 683             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 684                                /*writeTx*/null);
 685         }
 686     }
 687
 688
 689     /*
 690      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 691      */
 692     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 693                                                final VrfEntry vrfEntry, long vpnTag) {
 694         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 695         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 696             && vrfEntry.getRoutePaths().size() == 1);
 697         String destination = vrfEntry.getDestPrefix();
 698         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 699         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 700
 701         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 702         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 703         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 704         if (interVpnLink.getState().or(State.Error) != State.Active) {
 705             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 706                 destination, nextHop, interVpnLinkName);
 707             return;
 708         }
 709
 710         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 711         if (!optOtherEndpointLportTag.isPresent()) {
 712             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 713                 vpnUuid, interVpnLinkName);
 714             return;
 715         }
 716
 717         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 718         if (targetDpns.isEmpty()) {
 719             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 720                 vpnUuid, interVpnLinkName);
 721             return;
 722         }
 723
 724         String[] values = destination.split("/");
 725         String destPrefixIpAddress = values[0];
 726         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 727
 728         List<MatchInfo> matches = new ArrayList<>();
 729         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 730         matches.add(MatchEthernetType.IPV4);
 731
 732         if (prefixLength != 0) {
 733             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 734         }
 735
 736         List<Instruction> instructions =
 737             Arrays.asList(new InstructionWriteMetadata(
 738                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 739                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 740                             .L3VPN_SERVICE_INDEX)),
 741                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 742                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 743
 744         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 745         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 746         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 747             COOKIE_VM_FIB_TABLE, matches, instructions);
 748
 749         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 750             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 751
 752         for (BigInteger dpId : targetDpns) {
 753
 754             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 755                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 756                 dpId, interVpnLink.getInterVpnLinkName());
 757
 758             mdsalManager.installFlow(dpId, flowEntity);
 759         }
 760     }
 761
 762     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 763         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 764         String localNextHopIP = vrfEntry.getDestPrefix();
 765         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 766         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 767         if (localNextHopInfo == null) {
 768             boolean localNextHopSeen = false;
 769             List<Routes> vpnExtraRoutes = null;
 770             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 771             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 772             // FIXME: use an Identifier structure?
 773             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 774             lock.lock();
 775             try {
 776                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 777                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 778                         vpnName, usedRds, localNextHopIP);
 779                 if (LOG.isDebugEnabled()) {
 780                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 781                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 782                 }
 783
 784                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 785                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 786                     String ipPrefix;
 787                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 788                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 789                     } else {
 790                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 791                     }
 792                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 793                             ipPrefix);
 794                     if (localNextHopInfoLocal != null) {
 795                         localNextHopSeen = true;
 796                         BigInteger dpnId =
 797                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 798                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 799                         returnLocalDpnId.add(dpnId);
 800                     }
 801                 }
 802             } finally {
 803                 lock.unlock();
 804             }
 805             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 806                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 807                 if (optionalLabel.isPresent()) {
 808                     Long label = optionalLabel.get();
 809                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 810                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 811                     final ReentrantLock labelLock = lockFor(lriKey);
 812                     labelLock.lock();
 813                     try {
 814                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 815                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 816                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 817                                     fibUtil.getVpnInstanceOpData(rd);
 818                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 819                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 820                                 if (nullToEmpty(lri.getVpnInstanceList()).contains(vpnInstanceName)) {
 821                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 822                                     localNextHopIP = lri.getPrefix();
 823                                 } else {
 824                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 825                                     localNextHopIP = lri.getPrefix();
 826                                 }
 827                             }
 828                             if (localNextHopInfo != null) {
 829                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 830                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 831                                 if (vpnExtraRoutes.isEmpty()) {
 832                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 833                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 834                                     returnLocalDpnId.add(dpnId);
 835                                 } else {
 836                                     for (Routes extraRoutes : vpnExtraRoutes) {
 837                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 838                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 839                                         returnLocalDpnId.add(dpnId);
 840                                     }
 841                                 }
 842                             }
 843                         }
 844                     } finally {
 845                         labelLock.unlock();
 846                     }
 847                 }
 848             }
 849             if (returnLocalDpnId.isEmpty()) {
 850                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 851             }
 852         } else {
 853             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 854                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 855             if (dpnId != null) {
 856                 returnLocalDpnId.add(dpnId);
 857             }
 858         }
 859         return returnLocalDpnId;
 860     }
 861
 862     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 863                                                 final Long vpnId, final String rd,
 864                                                 final VrfEntry vrfEntry,
 865                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 866                                                 int etherType) {
 867         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 868         if (localNextHopInfo != null) {
 869             long groupId;
 870             long localGroupId;
 871             final BigInteger dpnId = localNextHopInfo.getDpnId();
 872             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 873                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 874                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 875                 return dpnId;
 876             }
 877             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 878                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 879                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 880                 return dpnId;
 881             }
 882             if (!isVpnPresentInDpn(rd, dpnId)) {
 883                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 884                         vpnId, rd, dpnId.toString());
 885                 return BigInteger.ZERO;
 886             }
 887             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 888             String prefix = vrfEntry.getDestPrefix();
 889             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 890             //The loadbalancing group is created only if the extra route has multiple nexthops
 891             //to avoid loadbalancing the discovered routes
 892             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 893                     && routes != null) {
 894                 if (vpnExtraRoutes.size() > 1) {
 895                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 896                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 897                 } else {
 898                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 899                     localGroupId = groupId;
 900                 }
 901             } else {
 902                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 903                         gwMacAddress);
 904                 localGroupId = groupId;
 905             }
 906             if (groupId == FibConstants.INVALID_GROUP_ID) {
 907                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 908                         prefix, rd, interfaceName, dpnId.toString());
 909                 return BigInteger.ZERO;
 910             }
 911             final List<InstructionInfo> instructions = Collections.singletonList(
 912                     new InstructionApplyActions(
 913                             Collections.singletonList(new ActionGroup(groupId))));
 914             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 915                     new InstructionApplyActions(
 916                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 917             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 918             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 919             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 920             jobCoordinator.enqueueJob(jobKey,
 921                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 922                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 923                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 924                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 925                         optLabel.ifPresent(label -> {
 926                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 927                                 LOG.debug(
 928                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 929                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 930                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 931                                         nextHopAddressList);
 932                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 933                                         NwConstants.ADD_FLOW, tx);
 934                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 935                             } else {
 936                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 937                                                 + "route. LFib and Terminating table entries will not be created.",
 938                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 939                             }
 940                         });
 941                     }
 942                 })));
 943             return dpnId;
 944         }
 945         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 946                 vpnName);
 947         return BigInteger.ZERO;
 948     }
 949
 950     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 951         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 952         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 953         return dpnInVpn.isPresent();
 954     }
 955
 956     @Nullable
 957     private LabelRouteInfo getLabelRouteInfo(Long label) {
 958         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 959     }
 960
 961     @Nullable
 962     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 963         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 964             .child(LabelRouteInfo.class, label).build();
 965         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 966         if (opResult.isPresent()) {
 967             return opResult.get();
 968         }
 969         return null;
 970     }
 971
 972     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 973             @Nullable TypedWriteTransaction<Operational> tx) {
 974         if (lri == null) {
 975             return true;
 976         }
 977
 978         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 979         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 980             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 981
 982         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 983             ? lri.getVpnInstanceList() : new ArrayList<>();
 984         if (vpnInstancesList.contains(vpnInstanceName)) {
 985             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 986             vpnInstancesList.remove(vpnInstanceName);
 987         }
 988         if (vpnInstancesList.isEmpty()) {
 989             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 990             if (tx != null) {
 991                 tx.delete(lriId);
 992             } else {
 993                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 994             }
 995             return true;
 996         } else {
 997             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 998             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 999             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1000         }
1001         return false;
1002     }
1003
1004     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
1005                                       TypedWriteTransaction<Configuration> tx) {
1006         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1007
1008         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
1009
1010         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1011             dpId, label, groupId);
1012     }
1013
1014     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
1015                                                 TypedWriteTransaction<Configuration> tx) {
1016         List<MatchInfo> mkMatches = new ArrayList<>();
1017
1018         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1019             destDpId, label, actionsInfos);
1020
1021         // Matching metadata
1022         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1023         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1024
1025         List<InstructionInfo> mkInstructions = new ArrayList<>();
1026         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1027
1028         FlowEntity terminatingServiceTableFlowEntity =
1029             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1030             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1031                 String.format("%s:%d", "TST Flow Entry ", label),
1032             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1033
1034         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1035
1036         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1037
1038         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1039         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1040             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1041             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1042             .child(Flow.class, flowKey).build();
1043         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1044     }
1045
1046     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1047         FlowEntity flowEntity;
1048         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1049         List<MatchInfo> mkMatches = new ArrayList<>();
1050         // Matching metadata
1051         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1052         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1053             NwConstants.INTERNAL_TUNNEL_TABLE,
1054             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1055             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1056             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1057         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1058         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1059         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1060             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1061             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1062
1063         tx.delete(flowInstanceId);
1064         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1065     }
1066
1067     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1068         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1069         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1070         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1071         boolean shouldUpdateNonEcmpLocalNextHop = true;
1072         if (localNextHopInfo == null) {
1073             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1074             if (usedRds.size() > 1) {
1075                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1076                         vrfEntry.getDestPrefix(), vpnName, rd);
1077                 return returnLocalDpnId;
1078             }
1079             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1080             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1081             //in the vpn
1082             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1083                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1084             if (extraRouteOptional.isPresent()) {
1085                 Routes extraRoute = extraRouteOptional.get();
1086                 String ipPrefix;
1087                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1088                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1089                 } else {
1090                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1091                 }
1092                 if (extraRoute.getNexthopIpList().size() > 1) {
1093                     shouldUpdateNonEcmpLocalNextHop = false;
1094                 }
1095                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1096                 if (localNextHopInfo != null) {
1097                     String localNextHopIP = localNextHopInfo.getIpAddress();
1098                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1099                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1100                     if (!dpnId.equals(BigInteger.ZERO)) {
1101                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1102                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1103                         returnLocalDpnId.add(dpnId);
1104                     }
1105                 } else {
1106                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1107                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1108                 }
1109             }
1110
1111             if (localNextHopInfo == null) {
1112                 /* Imported VRF entry */
1113                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1114                 if (optionalLabel.isPresent()) {
1115                     Long label = optionalLabel.get();
1116                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1117                     LabelRouteInfo lri = getLabelRouteInfo(label);
1118                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1119                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1120                         prefixBuilder.setDpnId(lri.getDpnId());
1121                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1122                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1123                         if (!dpnId.equals(BigInteger.ZERO)) {
1124                             returnLocalDpnId.add(dpnId);
1125                         }
1126                     }
1127                 }
1128             }
1129
1130         } else {
1131             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1132             String localNextHopIP = localNextHopInfo.getIpAddress();
1133             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1134                     shouldUpdateNonEcmpLocalNextHop);
1135             if (!dpnId.equals(BigInteger.ZERO)) {
1136                 returnLocalDpnId.add(dpnId);
1137             }
1138         }
1139
1140         return returnLocalDpnId;
1141     }
1142
1143     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1144             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1145             boolean shouldUpdateNonEcmpLocalNextHop) {
1146         if (localNextHopInfo != null) {
1147             final BigInteger dpnId = localNextHopInfo.getDpnId();
1148             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1149                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1150                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1151                 return dpnId;
1152             }
1153             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1154                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1155                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1156                 return dpnId;
1157             }
1158
1159             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1160                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1161                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1162                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1163                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1164                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1165                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1166                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1167                                         NwConstants.DEL_FLOW, tx);
1168                                 removeTunnelTableEntry(dpnId, label, tx);
1169                             });
1170                         }
1171                     }
1172                 })));
1173             //TODO: verify below adjacency call need to be optimized (?)
1174             //In case of the removal of the extra route, the loadbalancing group is updated
1175             if (shouldUpdateNonEcmpLocalNextHop) {
1176                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1177             }
1178             return dpnId;
1179         }
1180         return BigInteger.ZERO;
1181     }
1182
1183     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1184             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1185         if (tx == null) {
1186             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1187                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1188                 "Error creating remote FIB entry");
1189             return;
1190         }
1191
1192         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1193         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1194                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1195
1196         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1197         if (adjacencyResults.isEmpty()) {
1198             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1199                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1200             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1201             return;
1202         }
1203
1204         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1205         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1206                 vpnName, usedRds, vrfEntry.getDestPrefix());
1207         // create loadbalancing groups for extra routes only when the extra route is present behind
1208         // multiple VMs
1209         if (!vpnExtraRoutes.isEmpty()) {
1210             List<InstructionInfo> instructions = new ArrayList<>();
1211             // Obtain the local routes for this particular dpn.
1212             java.util.Optional<Routes> routes = vpnExtraRoutes
1213                     .stream()
1214                     .filter(route -> {
1215                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1216                                 FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1217                         if (prefixToInterface == null) {
1218                             return false;
1219                         }
1220                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1221                     }).findFirst();
1222             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1223                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1224             if (groupId == FibConstants.INVALID_GROUP_ID) {
1225                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1226                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1227                 return;
1228             }
1229             List<ActionInfo> actionInfos =
1230                     Collections.singletonList(new ActionGroup(groupId));
1231             instructions.add(new InstructionApplyActions(actionInfos));
1232             String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1233             jobCoordinator.enqueueJob(jobKey,
1234                 () ->  Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1235                     baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1236                             NwConstants.ADD_FLOW, txn, null);
1237                 })));
1238         } else {
1239             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry,
1240                 TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null);
1241         }
1242
1243         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1244     }
1245
1246     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1247     /* Get interface info from prefix to interface mapping;
1248         Use the interface info to get the corresponding vpn interface op DS entry,
1249         remove the adjacency corresponding to this fib entry.
1250         If adjacency removed is the last adjacency, clean up the following:
1251          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1252          - prefix to interface entry
1253          - vpn interface op DS
1254      */
1255         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1256         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1257         if (prefixInfo == null) {
1258             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1259             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1260             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1261             if (extraRoute != null) {
1262                 for (String nextHopIp : nullToEmpty(extraRoute.getNexthopIpList())) {
1263                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1264                     if (nextHopIp != null) {
1265                         String ipPrefix;
1266                         if (isIpv4Address(nextHopIp)) {
1267                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1268                         } else {
1269                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1270                         }
1271                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1272                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1273                     }
1274                 }
1275             }
1276             if (prefixInfo == null) {
1277                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1278                 if (optionalLabel.isPresent()) {
1279                     Long label = optionalLabel.get();
1280                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1281                     LabelRouteInfo lri = getLabelRouteInfo(label);
1282                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1283                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1284                         prefixBuilder.setDpnId(lri.getDpnId());
1285                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1286                         prefixBuilder.setIpAddress(lri.getPrefix());
1287                         prefixInfo = prefixBuilder.build();
1288                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1289                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1290                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1291                     }
1292                 }
1293             }
1294         } else {
1295             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1296         }
1297     }
1298
1299     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1300                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1301
1302         if (prefixInfo == null) {
1303             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1304                     vrfEntry.getDestPrefix(), vpnId, rd);
1305             return; //Don't have any info for this prefix (shouldn't happen); need to return
1306         }
1307
1308         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1309             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1310                     vrfEntry.getDestPrefix(), vpnId, rd);
1311             return;
1312         }
1313
1314         String ifName = prefixInfo.getVpnInterfaceName();
1315         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1316             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1317     }
1318
1319     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1320         Prefixes prefixInfo;
1321         Long vpnId;
1322         String rd;
1323         VrfEntry vrfEntry;
1324         Routes extraRoute;
1325
1326         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1327                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1328             this.prefixInfo = prefixInfo;
1329             this.vpnId = vpnId;
1330             this.rd = rd;
1331             this.vrfEntry = vrfEntry;
1332             this.extraRoute = extraRoute;
1333         }
1334
1335         @Override
1336         public List<ListenableFuture<Void>> call() {
1337             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1338             // to call the respective helpers.
1339             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1340                 //First Cleanup LabelRouteInfo
1341                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1342                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1343                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1344                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1345                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1346                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1347                         final ReentrantLock lock = lockFor(lriKey);
1348                         lock.lock();
1349                         try {
1350                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1351                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1352                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1353                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1354                                         fibUtil.getVpnInstanceOpData(rd);
1355                                 String vpnInstanceName = "";
1356                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1357                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1358                                 }
1359                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1360                                 if (lriRemoved) {
1361                                     String parentRd = lri.getParentVpnRd();
1362                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1363                                             parentRd, vrfEntry.getDestPrefix()));
1364                                 }
1365                             } else {
1366                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1367                                         rd, vrfEntry.getDestPrefix()));
1368                             }
1369                         } finally {
1370                             lock.unlock();
1371                         }
1372                     });
1373                 }
1374                 String ifName = prefixInfo.getVpnInterfaceName();
1375                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1376                 String vpnName = null;
1377
1378                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1379                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1380                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1381                     if (vpnNameOpt.isPresent()) {
1382                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1383                     }
1384                 }
1385                 if (optVpnName.isPresent()) {
1386                     vpnName = optVpnName.get();
1387                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1388                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1389                     if (opVpnInterface.isPresent()) {
1390                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1391                         if (vpnId != associatedVpnId) {
1392                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1393                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1394                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1395                             return;
1396                         } else {
1397                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1398                                     vrfEntry.getDestPrefix(), associatedVpnId);
1399                         }
1400                     }
1401                 }
1402                 if (extraRoute != null) {
1403                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1404                     //Only one used Rd present in case of removal event
1405                     String usedRd = usedRds.get(0);
1406                     if (optVpnName.isPresent()) {
1407                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1408                                         vrfEntry.getDestPrefix()));
1409                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1410                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1411                     }
1412                 }
1413                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1414             }));
1415         }
1416     }
1417
1418     /**
1419      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1420      * Remove Adjacency from VPNInterfaceOpData.
1421      * if Adjacency != primary.
1422      * if Adjacency == primary , then mark it for deletion.
1423      * Remove entire VPNinterfaceOpData Entry.
1424      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1425      * @param vrfEntry - VrfEntry removed
1426      * @param ifName - Interface name from VRFentry
1427      * @param vpnName - VPN name of corresponding VRF
1428      * @param tx - ReadWrite Tx
1429      */
1430     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1431                                                           TypedReadWriteTransaction<Operational> tx)
1432             throws ExecutionException, InterruptedException {
1433         InstanceIdentifier<Adjacency> adjacencyIid =
1434                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1435         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1436         if (adjacencyOptional.isPresent()) {
1437             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1438                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1439             } else {
1440                 tx.merge(adjacencyIid,
1441                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1442             }
1443         }
1444
1445         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1446
1447         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1448             return;
1449         }
1450
1451         if (nullToEmpty(optAdjacencies.get().getAdjacency()).size() <= 2
1452                 && nullToEmpty(optAdjacencies.get().getAdjacency()).stream().allMatch(adjacency ->
1453                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1454                         && adjacency.isMarkedForDeletion() != null
1455                         && adjacency.isMarkedForDeletion()
1456         )) {
1457             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1458             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1459         }
1460     }
1461
1462     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1463         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1464         final String rd = vrfTableKey.getRouteDistinguisher();
1465         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1466         if (vpnInstance == null) {
1467             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1468             return;
1469         }
1470         final Collection<VpnToDpnList> vpnToDpnList;
1471         if (vrfEntry.getParentVpnRd() != null
1472                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1473             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1474             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1475             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1476                     vpnInstance.getVpnToDpnList();
1477             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1478                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1479         } else {
1480             vpnToDpnList = vpnInstance.getVpnToDpnList();
1481         }
1482
1483         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1484         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1485         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1486         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1487         if (subnetRoute != null) {
1488             long elanTag = subnetRoute.getElantag();
1489             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1490                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1491             if (vpnToDpnList != null) {
1492                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1493                     () -> Collections.singletonList(
1494                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1495                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1496
1497                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1498                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1499                                     TransactionAdapter.toWriteTransaction(tx), null);
1500                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1501                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1502                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1503                                 }
1504
1505                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1506                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1507                             }
1508                         })));
1509             }
1510             optionalLabel.ifPresent(label -> {
1511                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1512                 final ReentrantLock lock = lockFor(lriKey);
1513                 lock.lock();
1514                 try {
1515                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1516                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1517                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1518                                 fibUtil.getVpnInstanceOpData(rd);
1519                         String vpnInstanceName = "";
1520                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1521                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1522                         }
1523                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1524                         if (lriRemoved) {
1525                             String parentRd = lri.getParentVpnRd();
1526                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1527                                     parentRd, vrfEntry.getDestPrefix()));
1528                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1529                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1530                         }
1531                     } else {
1532                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1533                                 rd, vrfEntry.getDestPrefix()));
1534                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1535                                 label, rd, vrfEntry.getDestPrefix());
1536                     }
1537                 } finally {
1538                     lock.unlock();
1539                 }
1540             });
1541             return;
1542         }
1543
1544         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1545             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1546         if (vpnToDpnList != null) {
1547             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1548                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1549             String jobKey;
1550             Optional<Routes> extraRouteOptional;
1551             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1552             if (usedRds != null && !usedRds.isEmpty()) {
1553                 if (usedRds.size() > 1) {
1554                     LOG.error("The extra route prefix is still present in some DPNs");
1555                     return ;
1556                 } else {
1557                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1558                     //is not present in any other DPN
1559                     extraRouteOptional = VpnExtraRouteHelper
1560                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1561                 }
1562             } else {
1563                 extraRouteOptional = Optional.absent();
1564             }
1565
1566             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1567                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1568                     if (localDpnIdList.size() <= 0) {
1569                         for (VpnToDpnList curDpn : vpnToDpnList) {
1570                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1571                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1572                                 TransactionAdapter.toWriteTransaction(tx));
1573                         }
1574                     } else {
1575                         for (BigInteger localDpnId : localDpnIdList) {
1576                             for (VpnToDpnList curDpn : vpnToDpnList) {
1577                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1578                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1579                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1580                                         TransactionAdapter.toWriteTransaction(tx));
1581                                 }
1582                             }
1583                         }
1584                     }
1585                     if (extraRouteOptional.isPresent()) {
1586                         //Remove select groups only for extra-routes
1587                         nextHopManager.removeNextHopPointer(nextHopManager
1588                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1589                         nextHopManager.removeNextHopPointer(nextHopManager
1590                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1591                     }
1592                 })), MAX_RETRIES);
1593         }
1594
1595         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1596         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1597         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1598
1599         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1600         // of the interVpnLink.
1601         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1602         if (optVpnUuid.isPresent()) {
1603             String vpnUuid = optVpnUuid.get();
1604             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1605                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1606                 if (optInterVpnLink.isPresent()) {
1607                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1608                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1609                         // This is route that points to the other endpoint of an InterVpnLink
1610                         // In that case, we should look for the FIB table pointing to
1611                         // LPortDispatcher table and remove it.
1612                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1613                     }
1614                 }
1615             });
1616         }
1617
1618     }
1619
1620     private void makeLFibTableEntry(BigInteger dpId, long label, @Nullable List<InstructionInfo> instructions,
1621                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1622         if (tx == null) {
1623             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1624                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1625                 "Error making LFIB table entry");
1626             return;
1627         }
1628
1629         List<MatchInfo> matches = new ArrayList<>();
1630         matches.add(MatchEthernetType.MPLS_UNICAST);
1631         matches.add(new MatchMplsLabel(label));
1632
1633         // Install the flow entry in L3_LFIB_TABLE
1634         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1635
1636         FlowEntity flowEntity;
1637         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1638             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1639         Flow flow = flowEntity.getFlowBuilder().build();
1640         String flowId = flowEntity.getFlowId();
1641         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1642         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1643         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1644             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1645             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1646
1647         if (addOrRemove == NwConstants.ADD_FLOW) {
1648             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1649         } else {
1650             tx.delete(flowInstanceId);
1651         }
1652
1653         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1654             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1655     }
1656
1657     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1658                                     final FutureCallback<List<Void>> callback) {
1659         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1660         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1661             () -> {
1662                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1663                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1664                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1665                         LogicalDatastoreType.CONFIGURATION, id);
1666                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1667                 if (!vrfTable.isPresent()) {
1668                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1669                     if (callback != null) {
1670                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1671                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1672                     }
1673                     return futures;
1674                 }
1675
1676                 final ReentrantLock lock = lockFor(vpnInstance);
1677                 lock.lock();
1678                 try {
1679                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1680                         for (final VrfEntry vrfEntry : nullToEmpty(vrfTable.get().getVrfEntry())) {
1681                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1682                             if (subnetRoute != null) {
1683                                 long elanTag = subnetRoute.getElantag();
1684                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1685                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1686                                         tx);
1687                                 continue;
1688                             }
1689                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1690                             if (routerInt != null) {
1691                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1692                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1693                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1694                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1695                                         NwConstants.ADD_FLOW);
1696                                 continue;
1697                             }
1698                             //Handle local flow creation for imports
1699                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1700                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1701                                 if (optionalLabel.isPresent()) {
1702                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1703                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1704                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1705                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1706                                             try {
1707                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1708                                                         vrfEntry.getDestPrefix());
1709                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1710                                             } catch (IllegalArgumentException ex) {
1711                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1712                                                         vrfEntry.getDestPrefix());
1713                                             }
1714                                             continue;
1715                                         }
1716                                     }
1717                                 }
1718                             }
1719                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1720                                     vrfEntry, dpnId);
1721                             if (shouldCreateRemoteFibEntry) {
1722                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1723                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1724                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1725                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1726                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1727                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1728                                 } else {
1729                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1730                                             vrfEntry, tx);
1731                                 }
1732                             }
1733                         }
1734                     }));
1735                     if (callback != null) {
1736                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1737                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1738                     }
1739                 } finally {
1740                     lock.unlock();
1741                 }
1742                 return futures;
1743             });
1744     }
1745
1746     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1747                                             final String localNextHopIp, final String remoteNextHopIp) {
1748         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1749             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1750         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1751         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1752         List<SubTransaction> txnObjects =  new ArrayList<>();
1753         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1754         if (vrfTable.isPresent()) {
1755             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1756                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1757                     final ReentrantLock lock = lockFor(vpnInstance);
1758                     lock.lock();
1759                     try {
1760                         nullToEmpty(vrfTable.get().getVrfEntry()).stream()
1761                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1762                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1763                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1764                     } finally {
1765                         lock.unlock();
1766                     }
1767                 })));
1768         }
1769     }
1770
1771     public void manageRemoteRouteOnDPN(final boolean action,
1772                                        final BigInteger localDpnId,
1773                                        final long vpnId,
1774                                        final String rd,
1775                                        final String destPrefix,
1776                                        final String destTepIp,
1777                                        final long label) {
1778         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1779
1780         if (vpnInstance == null) {
1781             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1782             return;
1783         }
1784
1785         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1786             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1787                 final ReentrantLock lock = lockFor(vpnInstance);
1788                 lock.lock();
1789                 try {
1790                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1791                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1792                     if (vrfEntry == null) {
1793                         return;
1794                     }
1795                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1796                             action, localDpnId, vpnId, rd, destPrefix);
1797                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1798                     VrfEntry modVrfEntry;
1799                     if (routePathList == null || routePathList.isEmpty()) {
1800                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1801                                 Collections.singletonList(destTepIp),
1802                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1803                     } else {
1804                         modVrfEntry = vrfEntry;
1805                     }
1806
1807                     if (action) {
1808                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1809                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1810                                 modVrfEntry, tx);
1811                     } else {
1812                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1813                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1814                                 vrfEntry.getDestPrefix());
1815                         if (usedRds.size() > 1) {
1816                             LOG.debug("The extra route prefix is still present in some DPNs");
1817                             return;
1818                         }
1819                         //Is this fib route an extra route? If yes, get the nexthop which would be
1820                         //an adjacency in the vpn
1821                         Optional<Routes> extraRouteOptional = Optional.absent();
1822                         if (usedRds.size() != 0) {
1823                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1824                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1825                                     usedRds.get(0), vrfEntry.getDestPrefix());
1826                         }
1827                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1828                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1829                     }
1830                 } finally {
1831                     lock.unlock();
1832                 }
1833             })));
1834     }
1835
1836     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1837                                  final FutureCallback<List<Void>> callback) {
1838         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1839         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1840             () -> {
1841                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1842                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1843                 List<SubTransaction> txnObjects =  new ArrayList<>();
1844                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1845                         LogicalDatastoreType.CONFIGURATION, id);
1846                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1847                 if (vrfTable.isPresent()) {
1848                     final ReentrantLock lock = lockFor(vpnInstance);
1849                     lock.lock();
1850                     try {
1851                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1852                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1853                             for (final VrfEntry vrfEntry : nullToEmpty(vrfTable.get().getVrfEntry())) {
1854                                 /* parentRd is only filled for external PNF cases where the interface on the external
1855                                  * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1856                                  * #fibUtil.isInterfacePresentInDpn().
1857                                 * */
1858                                 String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1859                                         : rd;
1860                                 /* Handle subnet routes here */
1861                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1862                                 if (subnetRoute != null && !fibUtil
1863                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1864                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1865                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1866                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1867                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1868                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1869                                     if (routePaths != null) {
1870                                         for (RoutePaths routePath : routePaths) {
1871                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1872                                                     DEFAULT_FIB_FLOW_PRIORITY,
1873                                                     NwConstants.DEL_FLOW, tx);
1874                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1875                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1876                                                     vrfEntry.getDestPrefix());
1877                                         }
1878                                     }
1879                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1880                                             NwConstants.DEL_FLOW, tx);
1881                                     continue;
1882                                 }
1883                                 // ping responder for router interfaces
1884                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1885                                 if (routerInt != null) {
1886                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1887                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1888                                             routerInt.getMacAddress());
1889                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1890                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1891                                             NwConstants.DEL_FLOW);
1892                                     continue;
1893                                 }
1894
1895                                 //Handle local flow deletion for imports
1896                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1897                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1898                                     if (optionalLabel.isPresent()) {
1899                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1900                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1901                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1902                                                 lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1903                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1904                                         }
1905                                     }
1906                                 }
1907
1908                                 // Passing null as we don't know the dpn
1909                                 // to which prefix is attached at this point
1910                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1911                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1912                                 Optional<Routes> extraRouteOptional;
1913                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1914                                 //an adjacency in the vpn
1915                                 if (usedRds != null && !usedRds.isEmpty()) {
1916                                     if (usedRds.size() > 1) {
1917                                         LOG.error("The extra route prefix is still present in some DPNs");
1918                                         return;
1919                                     } else {
1920                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1921                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1922
1923                                     }
1924                                 } else {
1925                                     extraRouteOptional = Optional.absent();
1926                                 }
1927                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1928                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1929                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1930                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1931                                 } else {
1932                                     if (subnetRoute == null || !fibUtil
1933                                             .isInterfacePresentInDpn(parentRd, dpnId)) {
1934                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1935                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1936                                             TransactionAdapter.toWriteTransaction(tx));
1937                                     }
1938                                 }
1939                             }
1940                         }));
1941                     } finally {
1942                         lock.unlock();
1943                     }
1944                     if (callback != null) {
1945                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1946                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1947                     }
1948                 } else {
1949                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1950                 }
1951                 return futures;
1952             });
1953
1954     }
1955
1956     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1957                                            final String localNextHopIp, final String remoteNextHopIp) {
1958         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1959                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1960             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1961         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1962         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1963         List<SubTransaction> txnObjects =  new ArrayList<>();
1964         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1965         if (vrfTable.isPresent()) {
1966             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1967                 () -> {
1968                     final ReentrantLock lock = lockFor(vpnInstance);
1969                     lock.lock();
1970                     try {
1971                         return Collections.singletonList(
1972                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1973                                 tx -> nullToEmpty(vrfTable.get().getVrfEntry()).stream()
1974                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1975                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1976                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1977                                         txnObjects))));
1978                     } finally {
1979                         lock.unlock();
1980                     }
1981                 });
1982         }
1983     }
1984
1985     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1986         InstanceIdentifierBuilder<VrfTables> idBuilder =
1987             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1988         return idBuilder.build();
1989     }
1990
1991     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1992         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1993                 .FLOWID_SEPARATOR + nextHop;
1994     }
1995
1996     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1997         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1998                 + tableMiss + FLOWID_PREFIX;
1999     }
2000
2001     @Nullable
2002     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2003         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2004             .child(VrfTables.class, new VrfTablesKey(rd))
2005             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2006         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2007         if (vrfEntry.isPresent()) {
2008             return vrfEntry.get();
2009         }
2010         return null;
2011     }
2012
2013     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2014                                              final String vpnName,
2015                                              final VrfEntry vrfEntry) {
2016         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2017
2018         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2019         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2020
2021         if (targetDpns.isEmpty()) {
2022             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2023             return;
2024         }
2025
2026         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2027         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2028
2029         // delete from FIB
2030         //
2031         optNextHop.ifPresent(nextHop -> {
2032             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2033             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2034             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2035                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2036
2037             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2038             for (BigInteger dpId : targetDpns) {
2039                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2040                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2041
2042                 mdsalManager.removeFlow(dpId, flow);
2043             }
2044         });
2045
2046         // delete from LFIB
2047         //
2048         optLabel.ifPresent(label -> {
2049             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2050
2051             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2052                 for (BigInteger dpId : targetDpns) {
2053                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2054                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2055                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
2056                             tx);
2057                 }
2058             }), LOG, "Error removing flows");
2059         });
2060     }
2061
2062     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2063             List<String> nextHopAddressList, LabelRouteInfo lri) {
2064         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2065                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2066     }
2067
2068     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2069         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2070             return true;
2071         }
2072
2073         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2074         if (prefix != null) {
2075             BigInteger prefixDpnId = prefix.getDpnId();
2076             if (dpnId.equals(prefixDpnId)) {
2077                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2078                         vrfEntry, dpnId);
2079                 return false;
2080             }
2081         }
2082         return true;
2083     }
2084
2085     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2086         // FIXME: use vpnInstance.key() instead?
2087         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2088     }
2089
2090     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2091         return JvmGlobalLocks.getLockFor(label);
2092     }
2093 }