fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.math.BigInteger;
  23 import java.net.InetAddress;
  24 import java.net.UnknownHostException;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Collection;
  28 import java.util.Collections;
  29 import java.util.List;
  30 import java.util.Objects;
  31 import java.util.concurrent.Callable;
  32 import java.util.concurrent.CopyOnWriteArrayList;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.locks.ReentrantLock;
  35 import javax.annotation.Nullable;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.Datastore.Operational;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  48 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  49 import org.opendaylight.genius.infra.TransactionAdapter;
  50 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  51 import org.opendaylight.genius.infra.TypedWriteTransaction;
  52 import org.opendaylight.genius.mdsalutil.ActionInfo;
  53 import org.opendaylight.genius.mdsalutil.FlowEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NWUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  66 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  72 import org.opendaylight.genius.utils.JvmGlobalLocks;
  73 import org.opendaylight.genius.utils.ServiceIndex;
  74 import org.opendaylight.genius.utils.batching.SubTransaction;
  75 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  76 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  77 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  78 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  79 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  85 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124
 125 @Singleton
 126 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 127
 128     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 129     private static final String FLOWID_PREFIX = "L3.";
 130     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 131     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 132     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 133     private static final int LFIB_INTERVPN_PRIORITY = 15;
 134     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 135     private static final int MAX_RETRIES = 3;
 136     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 137
 138     private final DataBroker dataBroker;
 139     private final ManagedNewTransactionRunner txRunner;
 140     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 141     private final IMdsalApiManager mdsalManager;
 142     private final NexthopManager nextHopManager;
 143     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 144     private final BaseVrfEntryHandler baseVrfEntryHandler;
 145     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 146     private final JobCoordinator jobCoordinator;
 147     private final IElanService elanManager;
 148     private final FibUtil fibUtil;
 149     private final InterVpnLinkCache interVpnLinkCache;
 150     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 151     private final UpgradeState upgradeState;
 152     private final DataTreeEventCallbackRegistrar eventCallbacks;
 153
 154     @Inject
 155     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 156                             final NexthopManager nexthopManager,
 157                             final IElanService elanManager,
 158                             final BaseVrfEntryHandler vrfEntryHandler,
 159                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 160                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 161                             final JobCoordinator jobCoordinator,
 162                             final FibUtil fibUtil,
 163                             final InterVpnLinkCache interVpnLinkCache,
 164                             final UpgradeState upgradeState,
 165                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 166         super(VrfEntry.class, VrfEntryListener.class);
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 170         this.mdsalManager = mdsalApiManager;
 171         this.nextHopManager = nexthopManager;
 172         this.elanManager = elanManager;
 173         this.baseVrfEntryHandler = vrfEntryHandler;
 174         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 175         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 176         this.jobCoordinator = jobCoordinator;
 177         this.fibUtil = fibUtil;
 178         this.interVpnLinkCache = interVpnLinkCache;
 179         this.upgradeState = upgradeState;
 180         this.eventCallbacks = eventCallbacks;
 181     }
 182
 183     @Override
 184     @PostConstruct
 185     public void init() {
 186         LOG.info("{} init", getClass().getSimpleName());
 187         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 188     }
 189
 190     @Override
 191     @SuppressWarnings("checkstyle:IllegalCatch")
 192     public void close() {
 193         closeables.forEach(c -> {
 194             try {
 195                 c.close();
 196             } catch (Exception e) {
 197                 LOG.warn("Error closing {}", c, e);
 198             }
 199         });
 200     }
 201
 202     @Override
 203     protected VrfEntryListener getDataTreeChangeListener() {
 204         return VrfEntryListener.this;
 205     }
 206
 207     @Override
 208     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 209         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 210     }
 211
 212     @Override
 213     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 214         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 215         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 216         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 217                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 218         addFibEntries(identifier, vrfEntry, rd);
 219         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 220                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 221     }
 222
 223     //This method is temporary. Eventually Factory design pattern will be used to get
 224     // right VrfEntryhandle and invoke its methods.
 225     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 226         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 227             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 228             return;
 229         }
 230         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 231             LOG.info("EVPN flows need to be programmed.");
 232             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 233                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 234             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 235             closeables.add(evpnVrfEntryHandler);
 236             return;
 237         }
 238         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 239         if (routerInt != null) {
 240             // ping responder for router interfaces
 241             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 242             return;
 243         }
 244         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 245             createFibEntries(identifier, vrfEntry);
 246             return;
 247         }
 248     }
 249
 250     @Override
 251     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 252         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 253         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 254         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 255                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 256         removeFibEntries(identifier, vrfEntry, rd);
 257         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 258             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 259     }
 260
 261     //This method is temporary. Eventually Factory design pattern will be used to get
 262     // right VrfEntryhandle and invoke its methods.
 263     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 264         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 265             LOG.info("EVPN flows to be deleted");
 266             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 267                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 268             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 269             closeables.add(evpnVrfEntryHandler);
 270             return;
 271         }
 272         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 273         if (routerInt != null) {
 274             // ping responder for router interfaces
 275             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 276             return;
 277         }
 278         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 279             deleteFibEntries(identifier, vrfEntry);
 280             return;
 281         }
 282         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 283             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 284             return;
 285         }
 286     }
 287
 288     @Override
 289     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 290     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 291     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 292     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 293         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 294         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 295         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 296                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 297         // Handle BGP Routes first
 298         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 299             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 300             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 301                     rd, update.getDestPrefix(), update.getRoutePaths());
 302             return;
 303         }
 304
 305         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 306             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 307             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 308             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 309
 310             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 311             //updated vrf entry route path is null. This can happen during tunnel events.
 312             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 313             List<String> usedRds = new ArrayList<>();
 314             if (optVpnInstance.isPresent()) {
 315                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 316                         update.getDestPrefix());
 317             }
 318             // If original VRF Entry had nexthop null , but update VRF Entry
 319             // has nexthop , route needs to be created on remote Dpns
 320             if (originalRoutePath == null || originalRoutePath.isEmpty()
 321                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 322                 // TODO(vivek): Though ugly, Not handling this code now, as each
 323                 // tep add event will invoke flow addition
 324                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 325                         + " This event is IGNORED here.", update.getDestPrefix());
 326                 return;
 327             }
 328
 329             // If original VRF Entry had valid nexthop , but update VRF Entry
 330             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 331             if (updateRoutePath == null || updateRoutePath.isEmpty()
 332                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 333                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 334                         + "This event is IGNORED here.", update.getDestPrefix());
 335                 return;
 336             }
 337             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 338             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 339             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 340             List<ListenableFuture<Void>> futures = new ArrayList<>();
 341             ListenableFuture<Void> configFuture =
 342                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 343                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 344                         nextHopsRemoved.parallelStream()
 345                             .forEach(nextHopRemoved -> {
 346                                 try {
 347                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 348                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 349                                 } catch (ExecutionException | InterruptedException e) {
 350                                     throw new RuntimeException(e);
 351                                 }
 352                             }))));
 353             futures.add(configFuture);
 354             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 355                 @Override
 356                 public void onSuccess(Void result) {
 357                     createFibEntries(identifier, update);
 358                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 359                             rd, update.getDestPrefix(), update.getRoutePaths());
 360                 }
 361
 362                 @Override
 363                 public void onFailure(Throwable throwable) {
 364                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 365                             update, throwable);
 366                 }
 367             }, MoreExecutors.directExecutor());
 368             return;
 369         }
 370
 371         //Handle all other routes only on a cluster reboot
 372         if (original.equals(update)) {
 373             //Reboot use-case
 374             createFibEntries(identifier, update);
 375             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 376                     rd, update.getDestPrefix(), update.getRoutePaths());
 377             return;
 378         }
 379
 380         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 381                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 382     }
 383
 384     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 385         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 386         List<SubTransaction> txnObjects =  new ArrayList<>();
 387         final VpnInstanceOpDataEntry vpnInstance =
 388                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 389         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 390         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 391                 + " has null vpnId!");
 392         final Collection<VpnToDpnList> vpnToDpnList;
 393         if (vrfEntry.getParentVpnRd() != null
 394                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 395             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 396             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 397             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 398                 vpnInstance.getVpnToDpnList();
 399             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 400                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 401         } else {
 402             vpnToDpnList = vpnInstance.getVpnToDpnList();
 403         }
 404         final Long vpnId = vpnInstance.getVpnId();
 405         final String rd = vrfTableKey.getRouteDistinguisher();
 406         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 407         if (subnetRoute != null) {
 408             final long elanTag = subnetRoute.getElantag();
 409             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 410                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 411             if (vpnToDpnList != null) {
 412                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 413                     () -> Collections.singletonList(
 414                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 415                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 416                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 417                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 418                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 419                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 420                                 }
 421                             }
 422                         })));
 423             }
 424             return;
 425         }
 426         // Get etherType value based on the IpPrefix address family type
 427         int etherType;
 428         try {
 429             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 430         } catch (IllegalArgumentException ex) {
 431             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 432             return;
 433         }
 434
 435         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 436         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 437             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 438                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 439                     final ReentrantLock lock = lockFor(vpnInstance);
 440                     lock.lock();
 441                     try {
 442                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 443                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 444                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 445                                     try {
 446                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 447                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 448                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 449                                                     TransactionAdapter.toWriteTransaction(tx),
 450                                                     txnObjects);
 451                                         } else {
 452                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 453                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 454                                         }
 455                                     } catch (NullPointerException e) {
 456                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 457                                                 vrfEntry.getDestPrefix(), e);
 458                                     }
 459                                 }
 460                             }
 461                         }
 462                     } finally {
 463                         lock.unlock();
 464                     }
 465                 })), MAX_RETRIES);
 466         }
 467
 468         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 469         if (optVpnUuid.isPresent()) {
 470             String vpnUuid = optVpnUuid.get();
 471             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 472             if (interVpnLink != null) {
 473                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 474                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 475                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 476                         // This is an static route that points to the other endpoint of an InterVpnLink
 477                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 478                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 479                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 480                     }
 481                 });
 482             }
 483         }
 484     }
 485
 486     void refreshFibTables(String rd, String prefix) {
 487         InstanceIdentifier<VrfEntry> vrfEntryId =
 488                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 489                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 490         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 491         if (vrfEntry.isPresent()) {
 492             createFibEntries(vrfEntryId, vrfEntry.get());
 493         }
 494     }
 495
 496     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 497         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 498         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 499         prefixBuilder.setDpnId(lri.getDpnId());
 500         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 501         prefixBuilder.setIpAddress(lri.getPrefix());
 502         // Increment the refCount here
 503         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 504             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 505         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 506         if (!isPresentInList) {
 507             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 508             List<String> vpnInstanceNames =
 509                 lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 510             vpnInstanceNames.add(vpnInstanceName);
 511             builder.setVpnInstanceList(vpnInstanceNames);
 512             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 513         } else {
 514             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 515         }
 516         return prefixBuilder.build();
 517     }
 518
 519     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 520             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 521         if (tx == null) {
 522             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 523                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 524                 "Error installing subnet route in FIB");
 525             return;
 526         }
 527         int etherType;
 528         try {
 529             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 530         } catch (IllegalArgumentException ex) {
 531             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 532             return;
 533         }
 534         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 535             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 536             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 537             final ReentrantLock lock = lockFor(lriKey);
 538             lock.lock();
 539             try {
 540                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 541                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 542
 543                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 544                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 545                                 fibUtil.getVpnInstanceOpData(rd);
 546                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 547                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 548                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 549                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 550                             }
 551                         }
 552                     }
 553                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 554                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 555                 }
 556             } finally {
 557                 lock.unlock();
 558             }
 559         });
 560         final List<InstructionInfo> instructions = new ArrayList<>();
 561         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 562             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 563         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 564         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 565         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 566                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 567         if (vrfEntry.getRoutePaths() != null) {
 568             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 569                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 570                     List<ActionInfo> actionsInfos = new ArrayList<>();
 571                     // reinitialize instructions list for LFIB Table
 572                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 573                     actionsInfos.add(new ActionPopMpls(etherType));
 574                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 575                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 576                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 577                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 578
 579                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 580                             NwConstants.ADD_FLOW, tx);
 581                 }
 582             }
 583         }
 584     }
 585
 586     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 587             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 588         List<MatchInfo> matches = new ArrayList<>();
 589
 590         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 591                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 592         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 593         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 594         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 595
 596         InetAddress destPrefix;
 597         try {
 598             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 599         } catch (UnknownHostException e) {
 600             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 601                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 602             return;
 603         }
 604
 605         // Match on VpnId and SubnetBroadCast IP address
 606         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 607         matches.add(MatchEthernetType.IPV4);
 608
 609         if (prefixLength != 0) {
 610             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 611         }
 612
 613         //Action is to drop the packet
 614         List<InstructionInfo> dropInstructions = new ArrayList<>();
 615         List<ActionInfo> actionsInfos = new ArrayList<>();
 616         actionsInfos.add(new ActionDrop());
 617         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 618
 619         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 620         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 621         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 622                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 623
 624         Flow flow = flowEntity.getFlowBuilder().build();
 625         String flowId = flowEntity.getFlowId();
 626         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 627         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 628
 629         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 630                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 631                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 632
 633         if (addOrRemove == NwConstants.ADD_FLOW) {
 634             tx.put(flowInstanceId,flow, true);
 635         } else {
 636             tx.delete(flowInstanceId);
 637         }
 638     }
 639
 640     /*
 641      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 642      * LportDispatcher table (via table 80)
 643      */
 644     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 645                                             final VrfEntry vrfEntry, int etherType) {
 646         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 647         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 648         // packet is commuted from Vpn2 to Vpn1.
 649         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 650         if (!interVpnLink.isActive()) {
 651             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 652                      interVpnLinkName, vrfEntry.getDestPrefix());
 653             return;
 654         }
 655
 656         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 657         if (!optLportTag.isPresent()) {
 658             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 659             return;
 660         }
 661
 662         Long lportTag = optLportTag.get();
 663         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 664         if (label == null) {
 665             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 666                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 667             return;
 668         }
 669         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 670         List<InstructionInfo> instructions = Arrays.asList(
 671             new InstructionApplyActions(actionsInfos),
 672             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 673                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 674                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 675                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 676             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 677         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 678         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 679
 680         for (BigInteger dpId : targetDpns) {
 681             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 682                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 683
 684             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 685                                /*writeTx*/null);
 686         }
 687     }
 688
 689
 690     /*
 691      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 692      */
 693     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 694                                                final VrfEntry vrfEntry, long vpnTag) {
 695         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 696         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 697             && vrfEntry.getRoutePaths().size() == 1);
 698         String destination = vrfEntry.getDestPrefix();
 699         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 700         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 701
 702         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 703         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 704         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 705         if (interVpnLink.getState().or(State.Error) != State.Active) {
 706             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 707                 destination, nextHop, interVpnLinkName);
 708             return;
 709         }
 710
 711         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 712         if (!optOtherEndpointLportTag.isPresent()) {
 713             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 714                 vpnUuid, interVpnLinkName);
 715             return;
 716         }
 717
 718         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 719         if (targetDpns.isEmpty()) {
 720             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 721                 vpnUuid, interVpnLinkName);
 722             return;
 723         }
 724
 725         String[] values = destination.split("/");
 726         String destPrefixIpAddress = values[0];
 727         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 728
 729         List<MatchInfo> matches = new ArrayList<>();
 730         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 731         matches.add(MatchEthernetType.IPV4);
 732
 733         if (prefixLength != 0) {
 734             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 735         }
 736
 737         List<Instruction> instructions =
 738             Arrays.asList(new InstructionWriteMetadata(
 739                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 740                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 741                             .L3VPN_SERVICE_INDEX)),
 742                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 743                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 744
 745         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 746         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 747         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 748             COOKIE_VM_FIB_TABLE, matches, instructions);
 749
 750         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 751             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 752
 753         for (BigInteger dpId : targetDpns) {
 754
 755             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 756                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 757                 dpId, interVpnLink.getInterVpnLinkName());
 758
 759             mdsalManager.installFlow(dpId, flowEntity);
 760         }
 761     }
 762
 763     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 764         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 765         String localNextHopIP = vrfEntry.getDestPrefix();
 766         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 767         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 768         if (localNextHopInfo == null) {
 769             boolean localNextHopSeen = false;
 770             List<Routes> vpnExtraRoutes = null;
 771             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 772             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 773             // FIXME: use an Identifier structure?
 774             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 775             lock.lock();
 776             try {
 777                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 778                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 779                         vpnName, usedRds, localNextHopIP);
 780                 if (LOG.isDebugEnabled()) {
 781                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 782                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 783                 }
 784
 785                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 786                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 787                     String ipPrefix;
 788                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 789                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 790                     } else {
 791                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 792                     }
 793                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 794                             ipPrefix);
 795                     if (localNextHopInfoLocal != null) {
 796                         localNextHopSeen = true;
 797                         BigInteger dpnId =
 798                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 799                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 800                         returnLocalDpnId.add(dpnId);
 801                     }
 802                 }
 803             } finally {
 804                 lock.unlock();
 805             }
 806             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 807                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 808                 if (optionalLabel.isPresent()) {
 809                     Long label = optionalLabel.get();
 810                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 811                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 812                     final ReentrantLock labelLock = lockFor(lriKey);
 813                     labelLock.lock();
 814                     try {
 815                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 816                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 817                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 818                                     fibUtil.getVpnInstanceOpData(rd);
 819                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 820                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 821                                 if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains(
 822                                        vpnInstanceName)) {
 823                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 824                                     localNextHopIP = lri.getPrefix();
 825                                 } else {
 826                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 827                                     localNextHopIP = lri.getPrefix();
 828                                 }
 829                             }
 830                             if (localNextHopInfo != null) {
 831                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 832                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 833                                 if (vpnExtraRoutes.isEmpty()) {
 834                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 835                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 836                                     returnLocalDpnId.add(dpnId);
 837                                 } else {
 838                                     for (Routes extraRoutes : vpnExtraRoutes) {
 839                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 840                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 841                                         returnLocalDpnId.add(dpnId);
 842                                     }
 843                                 }
 844                             }
 845                         }
 846                     } finally {
 847                         labelLock.unlock();
 848                     }
 849                 }
 850             }
 851             if (returnLocalDpnId.isEmpty()) {
 852                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 853             }
 854         } else {
 855             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 856                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 857             if (dpnId != null) {
 858                 returnLocalDpnId.add(dpnId);
 859             }
 860         }
 861         return returnLocalDpnId;
 862     }
 863
 864     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 865                                                 final Long vpnId, final String rd,
 866                                                 final VrfEntry vrfEntry,
 867                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 868                                                 int etherType) {
 869         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 870         if (localNextHopInfo != null) {
 871             long groupId;
 872             long localGroupId;
 873             final BigInteger dpnId = localNextHopInfo.getDpnId();
 874             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 875                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 876                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 877                 return dpnId;
 878             }
 879             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 880                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 881                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 882                 return dpnId;
 883             }
 884             if (!isVpnPresentInDpn(rd, dpnId)) {
 885                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 886                         vpnId, rd, dpnId.toString());
 887                 return BigInteger.ZERO;
 888             }
 889             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 890             String prefix = vrfEntry.getDestPrefix();
 891             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 892             //The loadbalancing group is created only if the extra route has multiple nexthops
 893             //to avoid loadbalancing the discovered routes
 894             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 895                     && routes != null) {
 896                 if (vpnExtraRoutes.size() > 1) {
 897                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 898                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 899                 } else {
 900                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 901                     localGroupId = groupId;
 902                 }
 903             } else {
 904                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 905                         gwMacAddress);
 906                 localGroupId = groupId;
 907             }
 908             if (groupId == FibConstants.INVALID_GROUP_ID) {
 909                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 910                         prefix, rd, interfaceName, dpnId.toString());
 911                 return BigInteger.ZERO;
 912             }
 913             final List<InstructionInfo> instructions = Collections.singletonList(
 914                     new InstructionApplyActions(
 915                             Collections.singletonList(new ActionGroup(groupId))));
 916             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 917                     new InstructionApplyActions(
 918                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 919             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 920             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 921             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 922             jobCoordinator.enqueueJob(jobKey,
 923                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 924                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 925                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 926                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 927                         optLabel.ifPresent(label -> {
 928                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 929                                 LOG.debug(
 930                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 931                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 932                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 933                                         nextHopAddressList);
 934                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 935                                         NwConstants.ADD_FLOW, tx);
 936                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 937                             } else {
 938                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 939                                                 + "route. LFib and Terminating table entries will not be created.",
 940                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 941                             }
 942                         });
 943                     }
 944                 })));
 945             return dpnId;
 946         }
 947         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 948                 vpnName);
 949         return BigInteger.ZERO;
 950     }
 951
 952     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 953         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 954         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 955         return dpnInVpn.isPresent();
 956     }
 957
 958     @Nullable
 959     private LabelRouteInfo getLabelRouteInfo(Long label) {
 960         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 961     }
 962
 963     @Nullable
 964     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 965         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 966             .child(LabelRouteInfo.class, label).build();
 967         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 968         if (opResult.isPresent()) {
 969             return opResult.get();
 970         }
 971         return null;
 972     }
 973
 974     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 975             @Nullable TypedWriteTransaction<Operational> tx) {
 976         if (lri == null) {
 977             return true;
 978         }
 979
 980         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 981         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 982             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 983
 984         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 985             ? lri.getVpnInstanceList() : new ArrayList<>();
 986         if (vpnInstancesList.contains(vpnInstanceName)) {
 987             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 988             vpnInstancesList.remove(vpnInstanceName);
 989         }
 990         if (vpnInstancesList.isEmpty()) {
 991             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 992             if (tx != null) {
 993                 tx.delete(lriId);
 994             } else {
 995                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 996             }
 997             return true;
 998         } else {
 999             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
1000             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
1001             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1002         }
1003         return false;
1004     }
1005
1006     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
1007                                       TypedWriteTransaction<Configuration> tx) {
1008         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1009
1010         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
1011
1012         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1013             dpId, label, groupId);
1014     }
1015
1016     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
1017                                                 TypedWriteTransaction<Configuration> tx) {
1018         List<MatchInfo> mkMatches = new ArrayList<>();
1019
1020         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1021             destDpId, label, actionsInfos);
1022
1023         // Matching metadata
1024         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1025         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1026
1027         List<InstructionInfo> mkInstructions = new ArrayList<>();
1028         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1029
1030         FlowEntity terminatingServiceTableFlowEntity =
1031             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1032             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1033                 String.format("%s:%d", "TST Flow Entry ", label),
1034             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1035
1036         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1037
1038         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1039
1040         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1041         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1042             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1043             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1044             .child(Flow.class, flowKey).build();
1045         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1046     }
1047
1048     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1049         FlowEntity flowEntity;
1050         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1051         List<MatchInfo> mkMatches = new ArrayList<>();
1052         // Matching metadata
1053         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1054         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1055             NwConstants.INTERNAL_TUNNEL_TABLE,
1056             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1057             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1058             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1059         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1060         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1061         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1062             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1063             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1064
1065         tx.delete(flowInstanceId);
1066         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1067     }
1068
1069     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1070         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1071         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1072         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1073         boolean shouldUpdateNonEcmpLocalNextHop = true;
1074         if (localNextHopInfo == null) {
1075             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1076             if (usedRds.size() > 1) {
1077                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1078                         vrfEntry.getDestPrefix(), vpnName, rd);
1079                 return returnLocalDpnId;
1080             }
1081             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1082             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1083             //in the vpn
1084             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1085                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1086             if (extraRouteOptional.isPresent()) {
1087                 Routes extraRoute = extraRouteOptional.get();
1088                 String ipPrefix;
1089                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1090                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1091                 } else {
1092                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1093                 }
1094                 if (extraRoute.getNexthopIpList().size() > 1) {
1095                     shouldUpdateNonEcmpLocalNextHop = false;
1096                 }
1097                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1098                 if (localNextHopInfo != null) {
1099                     String localNextHopIP = localNextHopInfo.getIpAddress();
1100                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1101                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1102                     if (!dpnId.equals(BigInteger.ZERO)) {
1103                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1104                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1105                         returnLocalDpnId.add(dpnId);
1106                     }
1107                 } else {
1108                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1109                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1110                 }
1111             }
1112
1113             if (localNextHopInfo == null) {
1114                 /* Imported VRF entry */
1115                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1116                 if (optionalLabel.isPresent()) {
1117                     Long label = optionalLabel.get();
1118                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1119                     LabelRouteInfo lri = getLabelRouteInfo(label);
1120                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1121                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1122                         prefixBuilder.setDpnId(lri.getDpnId());
1123                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1124                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1125                         if (!dpnId.equals(BigInteger.ZERO)) {
1126                             returnLocalDpnId.add(dpnId);
1127                         }
1128                     }
1129                 }
1130             }
1131
1132         } else {
1133             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1134             String localNextHopIP = localNextHopInfo.getIpAddress();
1135             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1136                     shouldUpdateNonEcmpLocalNextHop);
1137             if (!dpnId.equals(BigInteger.ZERO)) {
1138                 returnLocalDpnId.add(dpnId);
1139             }
1140         }
1141
1142         return returnLocalDpnId;
1143     }
1144
1145     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1146             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1147             boolean shouldUpdateNonEcmpLocalNextHop) {
1148         if (localNextHopInfo != null) {
1149             final BigInteger dpnId = localNextHopInfo.getDpnId();
1150             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1151                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1152                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1153                 return dpnId;
1154             }
1155             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1156                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1157                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1158                 return dpnId;
1159             }
1160
1161             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1162                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1163                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1164                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1165                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1166                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1167                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1168                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1169                                         NwConstants.DEL_FLOW, tx);
1170                                 removeTunnelTableEntry(dpnId, label, tx);
1171                             });
1172                         }
1173                     }
1174                 })));
1175             //TODO: verify below adjacency call need to be optimized (?)
1176             //In case of the removal of the extra route, the loadbalancing group is updated
1177             if (shouldUpdateNonEcmpLocalNextHop) {
1178                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1179             }
1180             return dpnId;
1181         }
1182         return BigInteger.ZERO;
1183     }
1184
1185     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1186             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1187         if (tx == null) {
1188             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1189                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1190                 "Error creating remote FIB entry");
1191             return;
1192         }
1193
1194         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1195         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1196                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1197
1198         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1199         if (adjacencyResults.isEmpty()) {
1200             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1201                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1202             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1203             return;
1204         }
1205
1206         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1207         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1208                 vpnName, usedRds, vrfEntry.getDestPrefix());
1209         // create loadbalancing groups for extra routes only when the extra route is present behind
1210         // multiple VMs
1211         if (!vpnExtraRoutes.isEmpty()) {
1212             List<InstructionInfo> instructions = new ArrayList<>();
1213             // Obtain the local routes for this particular dpn.
1214             java.util.Optional<Routes> routes = vpnExtraRoutes
1215                     .stream()
1216                     .filter(route -> {
1217                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1218                                 FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1219                         if (prefixToInterface == null) {
1220                             return false;
1221                         }
1222                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1223                     }).findFirst();
1224             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1225                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1226             if (groupId == FibConstants.INVALID_GROUP_ID) {
1227                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1228                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1229                 return;
1230             }
1231             List<ActionInfo> actionInfos =
1232                     Collections.singletonList(new ActionGroup(groupId));
1233             instructions.add(new InstructionApplyActions(actionInfos));
1234             String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1235             jobCoordinator.enqueueJob(jobKey,
1236                 () ->  Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1237                     baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1238                             NwConstants.ADD_FLOW, txn, null);
1239                 })));
1240         } else {
1241             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry,
1242                 TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null);
1243         }
1244
1245         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1246     }
1247
1248     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1249     /* Get interface info from prefix to interface mapping;
1250         Use the interface info to get the corresponding vpn interface op DS entry,
1251         remove the adjacency corresponding to this fib entry.
1252         If adjacency removed is the last adjacency, clean up the following:
1253          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1254          - prefix to interface entry
1255          - vpn interface op DS
1256      */
1257         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1258         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1259         if (prefixInfo == null) {
1260             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1261             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1262             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1263             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
1264                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1265                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1266                     if (nextHopIp != null) {
1267                         String ipPrefix;
1268                         if (isIpv4Address(nextHopIp)) {
1269                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1270                         } else {
1271                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1272                         }
1273                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1274                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1275                     }
1276                 }
1277             }
1278             if (prefixInfo == null) {
1279                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1280                 if (optionalLabel.isPresent()) {
1281                     Long label = optionalLabel.get();
1282                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1283                     LabelRouteInfo lri = getLabelRouteInfo(label);
1284                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1285                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1286                         prefixBuilder.setDpnId(lri.getDpnId());
1287                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1288                         prefixBuilder.setIpAddress(lri.getPrefix());
1289                         prefixInfo = prefixBuilder.build();
1290                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1291                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1292                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1293                     }
1294                 }
1295             }
1296         } else {
1297             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1298         }
1299     }
1300
1301     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1302                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1303
1304         if (prefixInfo == null) {
1305             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1306                     vrfEntry.getDestPrefix(), vpnId, rd);
1307             return; //Don't have any info for this prefix (shouldn't happen); need to return
1308         }
1309
1310         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1311             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1312                     vrfEntry.getDestPrefix(), vpnId, rd);
1313             return;
1314         }
1315
1316         String ifName = prefixInfo.getVpnInterfaceName();
1317         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1318             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1319     }
1320
1321     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1322         Prefixes prefixInfo;
1323         Long vpnId;
1324         String rd;
1325         VrfEntry vrfEntry;
1326         Routes extraRoute;
1327
1328         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1329                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1330             this.prefixInfo = prefixInfo;
1331             this.vpnId = vpnId;
1332             this.rd = rd;
1333             this.vrfEntry = vrfEntry;
1334             this.extraRoute = extraRoute;
1335         }
1336
1337         @Override
1338         public List<ListenableFuture<Void>> call() {
1339             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1340             // to call the respective helpers.
1341             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1342                 //First Cleanup LabelRouteInfo
1343                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1344                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1345                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1346                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1347                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1348                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1349                         final ReentrantLock lock = lockFor(lriKey);
1350                         lock.lock();
1351                         try {
1352                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1353                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1354                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1355                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1356                                         fibUtil.getVpnInstanceOpData(rd);
1357                                 String vpnInstanceName = "";
1358                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1359                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1360                                 }
1361                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1362                                 if (lriRemoved) {
1363                                     String parentRd = lri.getParentVpnRd();
1364                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1365                                             parentRd, vrfEntry.getDestPrefix()));
1366                                 }
1367                             } else {
1368                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1369                                         rd, vrfEntry.getDestPrefix()));
1370                             }
1371                         } finally {
1372                             lock.unlock();
1373                         }
1374                     });
1375                 }
1376                 String ifName = prefixInfo.getVpnInterfaceName();
1377                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1378                 String vpnName = null;
1379
1380                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1381                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1382                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1383                     if (vpnNameOpt.isPresent()) {
1384                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1385                     }
1386                 }
1387                 if (optVpnName.isPresent()) {
1388                     vpnName = optVpnName.get();
1389                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1390                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1391                     if (opVpnInterface.isPresent()) {
1392                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1393                         if (vpnId != associatedVpnId) {
1394                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1395                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1396                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1397                             return;
1398                         } else {
1399                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1400                                     vrfEntry.getDestPrefix(), associatedVpnId);
1401                         }
1402                     }
1403                 }
1404                 if (extraRoute != null) {
1405                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1406                     //Only one used Rd present in case of removal event
1407                     String usedRd = usedRds.get(0);
1408                     if (optVpnName.isPresent()) {
1409                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1410                                         vrfEntry.getDestPrefix()));
1411                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1412                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1413                     }
1414                 }
1415                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1416             }));
1417         }
1418     }
1419
1420     /**
1421      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1422      * Remove Adjacency from VPNInterfaceOpData.
1423      * if Adjacency != primary.
1424      * if Adjacency == primary , then mark it for deletion.
1425      * Remove entire VPNinterfaceOpData Entry.
1426      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1427      * @param vrfEntry - VrfEntry removed
1428      * @param ifName - Interface name from VRFentry
1429      * @param vpnName - VPN name of corresponding VRF
1430      * @param tx - ReadWrite Tx
1431      */
1432     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1433                                                           TypedReadWriteTransaction<Operational> tx)
1434             throws ExecutionException, InterruptedException {
1435         InstanceIdentifier<Adjacency> adjacencyIid =
1436                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1437         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1438         if (adjacencyOptional.isPresent()) {
1439             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1440                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1441             } else {
1442                 tx.merge(adjacencyIid,
1443                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1444             }
1445         }
1446
1447         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1448
1449         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1450             return;
1451         }
1452
1453         @NonNull List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
1454         if (adjacencies.size() <= 2
1455                 && adjacencies.stream().allMatch(adjacency ->
1456                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1457                         && adjacency.isMarkedForDeletion() != null
1458                         && adjacency.isMarkedForDeletion()
1459         )) {
1460             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1461             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1462         }
1463     }
1464
1465     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1466         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1467         final String rd = vrfTableKey.getRouteDistinguisher();
1468         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1469         if (vpnInstance == null) {
1470             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1471             return;
1472         }
1473         final Collection<VpnToDpnList> vpnToDpnList;
1474         if (vrfEntry.getParentVpnRd() != null
1475                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1476             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1477             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1478             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1479                     vpnInstance.getVpnToDpnList();
1480             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1481                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1482         } else {
1483             vpnToDpnList = vpnInstance.getVpnToDpnList();
1484         }
1485
1486         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1487         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1488         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1489         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1490         if (subnetRoute != null) {
1491             long elanTag = subnetRoute.getElantag();
1492             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1493                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1494             if (vpnToDpnList != null) {
1495                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1496                     () -> Collections.singletonList(
1497                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1498                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1499
1500                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1501                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1502                                     TransactionAdapter.toWriteTransaction(tx), null);
1503                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1504                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1505                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1506                                 }
1507
1508                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1509                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1510                             }
1511                         })));
1512             }
1513             optionalLabel.ifPresent(label -> {
1514                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1515                 final ReentrantLock lock = lockFor(lriKey);
1516                 lock.lock();
1517                 try {
1518                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1519                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1520                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1521                                 fibUtil.getVpnInstanceOpData(rd);
1522                         String vpnInstanceName = "";
1523                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1524                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1525                         }
1526                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1527                         if (lriRemoved) {
1528                             String parentRd = lri.getParentVpnRd();
1529                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1530                                     parentRd, vrfEntry.getDestPrefix()));
1531                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1532                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1533                         }
1534                     } else {
1535                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1536                                 rd, vrfEntry.getDestPrefix()));
1537                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1538                                 label, rd, vrfEntry.getDestPrefix());
1539                     }
1540                 } finally {
1541                     lock.unlock();
1542                 }
1543             });
1544             return;
1545         }
1546
1547         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1548             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1549         if (vpnToDpnList != null) {
1550             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1551                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1552             String jobKey;
1553             Optional<Routes> extraRouteOptional;
1554             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1555             if (usedRds != null && !usedRds.isEmpty()) {
1556                 if (usedRds.size() > 1) {
1557                     LOG.error("The extra route prefix is still present in some DPNs");
1558                     return ;
1559                 } else {
1560                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1561                     //is not present in any other DPN
1562                     extraRouteOptional = VpnExtraRouteHelper
1563                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1564                 }
1565             } else {
1566                 extraRouteOptional = Optional.absent();
1567             }
1568
1569             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1570                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1571                     if (localDpnIdList.size() <= 0) {
1572                         for (VpnToDpnList curDpn : vpnToDpnList) {
1573                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1574                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1575                                 TransactionAdapter.toWriteTransaction(tx));
1576                         }
1577                     } else {
1578                         for (BigInteger localDpnId : localDpnIdList) {
1579                             for (VpnToDpnList curDpn : vpnToDpnList) {
1580                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1581                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1582                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1583                                         TransactionAdapter.toWriteTransaction(tx));
1584                                 }
1585                             }
1586                         }
1587                     }
1588                     if (extraRouteOptional.isPresent()) {
1589                         //Remove select groups only for extra-routes
1590                         nextHopManager.removeNextHopPointer(nextHopManager
1591                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1592                         nextHopManager.removeNextHopPointer(nextHopManager
1593                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1594                     }
1595                 })), MAX_RETRIES);
1596         }
1597
1598         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1599         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1600         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1601
1602         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1603         // of the interVpnLink.
1604         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1605         if (optVpnUuid.isPresent()) {
1606             String vpnUuid = optVpnUuid.get();
1607             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1608                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1609                 if (optInterVpnLink.isPresent()) {
1610                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1611                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1612                         // This is route that points to the other endpoint of an InterVpnLink
1613                         // In that case, we should look for the FIB table pointing to
1614                         // LPortDispatcher table and remove it.
1615                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1616                     }
1617                 }
1618             });
1619         }
1620
1621     }
1622
1623     private void makeLFibTableEntry(BigInteger dpId, long label, @Nullable List<InstructionInfo> instructions,
1624                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1625         if (tx == null) {
1626             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1627                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1628                 "Error making LFIB table entry");
1629             return;
1630         }
1631
1632         List<MatchInfo> matches = new ArrayList<>();
1633         matches.add(MatchEthernetType.MPLS_UNICAST);
1634         matches.add(new MatchMplsLabel(label));
1635
1636         // Install the flow entry in L3_LFIB_TABLE
1637         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1638
1639         FlowEntity flowEntity;
1640         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1641             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1642         Flow flow = flowEntity.getFlowBuilder().build();
1643         String flowId = flowEntity.getFlowId();
1644         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1645         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1646         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1647             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1648             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1649
1650         if (addOrRemove == NwConstants.ADD_FLOW) {
1651             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1652         } else {
1653             tx.delete(flowInstanceId);
1654         }
1655
1656         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1657             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1658     }
1659
1660     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1661                                     final FutureCallback<List<Void>> callback) {
1662         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1663         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1664             () -> {
1665                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1666                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1667                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1668                         LogicalDatastoreType.CONFIGURATION, id);
1669                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1670                 if (!vrfTable.isPresent()) {
1671                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1672                     if (callback != null) {
1673                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1674                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1675                     }
1676                     return futures;
1677                 }
1678
1679                 final ReentrantLock lock = lockFor(vpnInstance);
1680                 lock.lock();
1681                 try {
1682                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1683                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1684                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1685                             if (subnetRoute != null) {
1686                                 long elanTag = subnetRoute.getElantag();
1687                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1688                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1689                                         tx);
1690                                 continue;
1691                             }
1692                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1693                             if (routerInt != null) {
1694                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1695                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1696                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1697                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1698                                         NwConstants.ADD_FLOW);
1699                                 continue;
1700                             }
1701                             //Handle local flow creation for imports
1702                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1703                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1704                                 if (optionalLabel.isPresent()) {
1705                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1706                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1707                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1708                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1709                                             try {
1710                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1711                                                         vrfEntry.getDestPrefix());
1712                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1713                                             } catch (IllegalArgumentException ex) {
1714                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1715                                                         vrfEntry.getDestPrefix());
1716                                             }
1717                                             continue;
1718                                         }
1719                                     }
1720                                 }
1721                             }
1722                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1723                                     vrfEntry, dpnId);
1724                             if (shouldCreateRemoteFibEntry) {
1725                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1726                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1727                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1728                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1729                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1730                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1731                                 } else {
1732                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1733                                             vrfEntry, tx);
1734                                 }
1735                             }
1736                         }
1737                     }));
1738                     if (callback != null) {
1739                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1740                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1741                     }
1742                 } finally {
1743                     lock.unlock();
1744                 }
1745                 return futures;
1746             });
1747     }
1748
1749     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1750                                             final String localNextHopIp, final String remoteNextHopIp) {
1751         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1752             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1753         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1754         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1755         List<SubTransaction> txnObjects =  new ArrayList<>();
1756         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1757         if (vrfTable.isPresent()) {
1758             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1759                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1760                     final ReentrantLock lock = lockFor(vpnInstance);
1761                     lock.lock();
1762                     try {
1763                         vrfTable.get().nonnullVrfEntry().stream()
1764                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1765                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1766                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1767                     } finally {
1768                         lock.unlock();
1769                     }
1770                 })));
1771         }
1772     }
1773
1774     public void manageRemoteRouteOnDPN(final boolean action,
1775                                        final BigInteger localDpnId,
1776                                        final long vpnId,
1777                                        final String rd,
1778                                        final String destPrefix,
1779                                        final String destTepIp,
1780                                        final long label) {
1781         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1782
1783         if (vpnInstance == null) {
1784             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1785             return;
1786         }
1787
1788         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1789             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1790                 final ReentrantLock lock = lockFor(vpnInstance);
1791                 lock.lock();
1792                 try {
1793                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1794                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1795                     if (vrfEntry == null) {
1796                         return;
1797                     }
1798                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1799                             action, localDpnId, vpnId, rd, destPrefix);
1800                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1801                     VrfEntry modVrfEntry;
1802                     if (routePathList == null || routePathList.isEmpty()) {
1803                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1804                                 Collections.singletonList(destTepIp),
1805                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1806                     } else {
1807                         modVrfEntry = vrfEntry;
1808                     }
1809
1810                     if (action) {
1811                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1812                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1813                                 modVrfEntry, tx);
1814                     } else {
1815                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1816                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1817                                 vrfEntry.getDestPrefix());
1818                         if (usedRds.size() > 1) {
1819                             LOG.debug("The extra route prefix is still present in some DPNs");
1820                             return;
1821                         }
1822                         //Is this fib route an extra route? If yes, get the nexthop which would be
1823                         //an adjacency in the vpn
1824                         Optional<Routes> extraRouteOptional = Optional.absent();
1825                         if (usedRds.size() != 0) {
1826                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1827                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1828                                     usedRds.get(0), vrfEntry.getDestPrefix());
1829                         }
1830                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1831                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1832                     }
1833                 } finally {
1834                     lock.unlock();
1835                 }
1836             })));
1837     }
1838
1839     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1840                                  final FutureCallback<List<Void>> callback) {
1841         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1842         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1843             () -> {
1844                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1845                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1846                 List<SubTransaction> txnObjects =  new ArrayList<>();
1847                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1848                         LogicalDatastoreType.CONFIGURATION, id);
1849                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1850                 if (vrfTable.isPresent()) {
1851                     final ReentrantLock lock = lockFor(vpnInstance);
1852                     lock.lock();
1853                     try {
1854                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1855                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1856                             for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1857                                 /* parentRd is only filled for external PNF cases where the interface on the external
1858                                  * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1859                                  * #fibUtil.isInterfacePresentInDpn().
1860                                 * */
1861                                 String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1862                                         : rd;
1863                                 /* Handle subnet routes here */
1864                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1865                                 if (subnetRoute != null && !fibUtil
1866                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1867                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1868                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1869                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1870                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1871                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1872                                     if (routePaths != null) {
1873                                         for (RoutePaths routePath : routePaths) {
1874                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1875                                                     DEFAULT_FIB_FLOW_PRIORITY,
1876                                                     NwConstants.DEL_FLOW, tx);
1877                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1878                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1879                                                     vrfEntry.getDestPrefix());
1880                                         }
1881                                     }
1882                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1883                                             NwConstants.DEL_FLOW, tx);
1884                                     continue;
1885                                 }
1886                                 // ping responder for router interfaces
1887                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1888                                 if (routerInt != null) {
1889                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1890                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1891                                             routerInt.getMacAddress());
1892                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1893                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1894                                             NwConstants.DEL_FLOW);
1895                                     continue;
1896                                 }
1897
1898                                 //Handle local flow deletion for imports
1899                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1900                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1901                                     if (optionalLabel.isPresent()) {
1902                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1903                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1904                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1905                                                 lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1906                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1907                                         }
1908                                     }
1909                                 }
1910
1911                                 // Passing null as we don't know the dpn
1912                                 // to which prefix is attached at this point
1913                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1914                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1915                                 Optional<Routes> extraRouteOptional;
1916                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1917                                 //an adjacency in the vpn
1918                                 if (usedRds != null && !usedRds.isEmpty()) {
1919                                     if (usedRds.size() > 1) {
1920                                         LOG.error("The extra route prefix is still present in some DPNs");
1921                                         return;
1922                                     } else {
1923                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1924                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1925
1926                                     }
1927                                 } else {
1928                                     extraRouteOptional = Optional.absent();
1929                                 }
1930                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1931                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1932                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1933                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1934                                 } else {
1935                                     if (subnetRoute == null || !fibUtil
1936                                             .isInterfacePresentInDpn(parentRd, dpnId)) {
1937                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1938                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1939                                             TransactionAdapter.toWriteTransaction(tx));
1940                                     }
1941                                 }
1942                             }
1943                         }));
1944                     } finally {
1945                         lock.unlock();
1946                     }
1947                     if (callback != null) {
1948                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1949                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1950                     }
1951                 } else {
1952                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1953                 }
1954                 return futures;
1955             });
1956
1957     }
1958
1959     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1960                                            final String localNextHopIp, final String remoteNextHopIp) {
1961         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1962                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1963             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1964         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1965         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1966         List<SubTransaction> txnObjects =  new ArrayList<>();
1967         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1968         if (vrfTable.isPresent()) {
1969             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1970                 () -> {
1971                     final ReentrantLock lock = lockFor(vpnInstance);
1972                     lock.lock();
1973                     try {
1974                         return Collections.singletonList(
1975                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1976                                 tx -> vrfTable.get().nonnullVrfEntry().stream()
1977                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1978                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1979                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1980                                         txnObjects))));
1981                     } finally {
1982                         lock.unlock();
1983                     }
1984                 });
1985         }
1986     }
1987
1988     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1989         InstanceIdentifierBuilder<VrfTables> idBuilder =
1990             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1991         return idBuilder.build();
1992     }
1993
1994     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1995         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1996                 .FLOWID_SEPARATOR + nextHop;
1997     }
1998
1999     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2000         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2001                 + tableMiss + FLOWID_PREFIX;
2002     }
2003
2004     @Nullable
2005     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2006         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2007             .child(VrfTables.class, new VrfTablesKey(rd))
2008             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2009         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2010         if (vrfEntry.isPresent()) {
2011             return vrfEntry.get();
2012         }
2013         return null;
2014     }
2015
2016     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2017                                              final String vpnName,
2018                                              final VrfEntry vrfEntry) {
2019         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2020
2021         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2022         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2023
2024         if (targetDpns.isEmpty()) {
2025             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2026             return;
2027         }
2028
2029         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2030         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2031
2032         // delete from FIB
2033         //
2034         optNextHop.ifPresent(nextHop -> {
2035             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2036             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2037             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2038                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2039
2040             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2041             for (BigInteger dpId : targetDpns) {
2042                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2043                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2044
2045                 mdsalManager.removeFlow(dpId, flow);
2046             }
2047         });
2048
2049         // delete from LFIB
2050         //
2051         optLabel.ifPresent(label -> {
2052             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2053
2054             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2055                 for (BigInteger dpId : targetDpns) {
2056                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2057                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2058                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
2059                             tx);
2060                 }
2061             }), LOG, "Error removing flows");
2062         });
2063     }
2064
2065     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2066             List<String> nextHopAddressList, LabelRouteInfo lri) {
2067         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2068                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2069     }
2070
2071     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2072         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2073             return true;
2074         }
2075
2076         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2077         if (prefix != null) {
2078             BigInteger prefixDpnId = prefix.getDpnId();
2079             if (dpnId.equals(prefixDpnId)) {
2080                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2081                         vrfEntry, dpnId);
2082                 return false;
2083             }
2084         }
2085         return true;
2086     }
2087
2088     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2089         // FIXME: use vpnInstance.key() instead?
2090         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2091     }
2092
2093     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2094         return JvmGlobalLocks.getLockFor(label);
2095     }
2096 }