fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.net.InetAddress;
  25 import java.net.UnknownHostException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.List;
  31 import java.util.Objects;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.locks.ReentrantLock;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.Datastore.Operational;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.TransactionAdapter;
  51 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  52 import org.opendaylight.genius.infra.TypedWriteTransaction;
  53 import org.opendaylight.genius.mdsalutil.ActionInfo;
  54 import org.opendaylight.genius.mdsalutil.FlowEntity;
  55 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  56 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  57 import org.opendaylight.genius.mdsalutil.MatchInfo;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NWUtil;
  60 import org.opendaylight.genius.mdsalutil.NwConstants;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  73 import org.opendaylight.genius.utils.JvmGlobalLocks;
  74 import org.opendaylight.genius.utils.ServiceIndex;
  75 import org.opendaylight.genius.utils.batching.SubTransaction;
  76 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  77 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  78 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  79 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  80 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  81 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  85 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  86 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127
 128 @Singleton
 129 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 130
 131     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 132     private static final String FLOWID_PREFIX = "L3.";
 133     private static final Uint64 COOKIE_VM_FIB_TABLE =  Uint64.valueOf("8000003", 16).intern();
 134     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 135     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 136     private static final int LFIB_INTERVPN_PRIORITY = 15;
 137     public static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 138     private static final int MAX_RETRIES = 3;
 139     private static final Uint64 COOKIE_TABLE_MISS = Uint64.valueOf("8000004", 16).intern();
 140
 141     private final DataBroker dataBroker;
 142     private final ManagedNewTransactionRunner txRunner;
 143     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 144     private final IMdsalApiManager mdsalManager;
 145     private final NexthopManager nextHopManager;
 146     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 147     private final BaseVrfEntryHandler baseVrfEntryHandler;
 148     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 149     private final JobCoordinator jobCoordinator;
 150     private final IElanService elanManager;
 151     private final FibUtil fibUtil;
 152     private final InterVpnLinkCache interVpnLinkCache;
 153     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 154     private final UpgradeState upgradeState;
 155     private final DataTreeEventCallbackRegistrar eventCallbacks;
 156
 157     @Inject
 158     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 159                             final NexthopManager nexthopManager,
 160                             final IElanService elanManager,
 161                             final BaseVrfEntryHandler vrfEntryHandler,
 162                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 163                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 164                             final JobCoordinator jobCoordinator,
 165                             final FibUtil fibUtil,
 166                             final InterVpnLinkCache interVpnLinkCache,
 167                             final UpgradeState upgradeState,
 168                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 169         super(VrfEntry.class, VrfEntryListener.class);
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 173         this.mdsalManager = mdsalApiManager;
 174         this.nextHopManager = nexthopManager;
 175         this.elanManager = elanManager;
 176         this.baseVrfEntryHandler = vrfEntryHandler;
 177         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 178         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 179         this.jobCoordinator = jobCoordinator;
 180         this.fibUtil = fibUtil;
 181         this.interVpnLinkCache = interVpnLinkCache;
 182         this.upgradeState = upgradeState;
 183         this.eventCallbacks = eventCallbacks;
 184     }
 185
 186     @Override
 187     @PostConstruct
 188     public void init() {
 189         LOG.info("{} init", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     @SuppressWarnings("checkstyle:IllegalCatch")
 195     public void close() {
 196         closeables.forEach(c -> {
 197             try {
 198                 c.close();
 199             } catch (Exception e) {
 200                 LOG.warn("Error closing {}", c, e);
 201             }
 202         });
 203     }
 204
 205     @Override
 206     protected VrfEntryListener getDataTreeChangeListener() {
 207         return VrfEntryListener.this;
 208     }
 209
 210     @Override
 211     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 212         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 213     }
 214
 215     @Override
 216     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 217         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 218         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 219         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 220                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 221         addFibEntries(identifier, vrfEntry, rd);
 222         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 223                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 224     }
 225
 226     //This method is temporary. Eventually Factory design pattern will be used to get
 227     // right VrfEntryhandle and invoke its methods.
 228     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 229         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 230             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 234             LOG.info("EVPN flows need to be programmed.");
 235             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 236                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 237             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 238             closeables.add(evpnVrfEntryHandler);
 239             return;
 240         }
 241         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 242         if (routerInt != null) {
 243             // ping responder for router interfaces
 244             routerInterfaceVrfEntryHandler.createFlows(vrfEntry, rd);
 245             return;
 246         }
 247         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 248             createFibEntries(identifier, vrfEntry);
 249             return;
 250         }
 251     }
 252
 253     @Override
 254     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 255         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 256         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 257         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 258                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 259         removeFibEntries(identifier, vrfEntry, rd);
 260         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 261             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 262     }
 263
 264     //This method is temporary. Eventually Factory design pattern will be used to get
 265     // right VrfEntryhandle and invoke its methods.
 266     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 267         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 268             LOG.info("EVPN flows to be deleted");
 269             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 270                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 271             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 272             closeables.add(evpnVrfEntryHandler);
 273             return;
 274         }
 275         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 276         if (routerInt != null) {
 277             // ping responder for router interfaces
 278             routerInterfaceVrfEntryHandler.removeFlows(vrfEntry, rd);
 279             return;
 280         }
 281         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 282             deleteFibEntries(identifier, vrfEntry);
 283             return;
 284         }
 285         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 286             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 287             return;
 288         }
 289     }
 290
 291     @Override
 292     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 293     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 294     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 295     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 296         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 297         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 298         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 299                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 300         // Handle BGP Routes first
 301         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 302             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 303             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 304                     rd, update.getDestPrefix(), update.getRoutePaths());
 305             return;
 306         }
 307
 308         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 309             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 310             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 311             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 312
 313             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 314             //updated vrf entry route path is null. This can happen during tunnel events.
 315             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 316             List<String> usedRds = new ArrayList<>();
 317             if (optVpnInstance.isPresent()) {
 318                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 319                         update.getDestPrefix());
 320             }
 321             // If original VRF Entry had nexthop null , but update VRF Entry
 322             // has nexthop , route needs to be created on remote Dpns
 323             if (originalRoutePath == null || originalRoutePath.isEmpty()
 324                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 325                 // TODO(vivek): Though ugly, Not handling this code now, as each
 326                 // tep add event will invoke flow addition
 327                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 328                         + " This event is IGNORED here.", update.getDestPrefix());
 329                 return;
 330             }
 331
 332             // If original VRF Entry had valid nexthop , but update VRF Entry
 333             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 334             if (updateRoutePath == null || updateRoutePath.isEmpty()
 335                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 336                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 337                         + "This event is IGNORED here.", update.getDestPrefix());
 338                 return;
 339             }
 340             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 341             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 342             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 343             List<ListenableFuture<Void>> futures = new ArrayList<>();
 344             ListenableFuture<Void> configFuture =
 345                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 346                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 347                         nextHopsRemoved.parallelStream()
 348                             .forEach(nextHopRemoved -> {
 349                                 try {
 350                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 351                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 352                                 } catch (ExecutionException | InterruptedException e) {
 353                                     throw new RuntimeException(e);
 354                                 }
 355                             }))));
 356             futures.add(configFuture);
 357             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 358                 @Override
 359                 public void onSuccess(Void result) {
 360                     createFibEntries(identifier, update);
 361                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 362                             rd, update.getDestPrefix(), update.getRoutePaths());
 363                 }
 364
 365                 @Override
 366                 public void onFailure(Throwable throwable) {
 367                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 368                             update, throwable);
 369                 }
 370             }, MoreExecutors.directExecutor());
 371             return;
 372         }
 373
 374         //Handle all other routes only on a cluster reboot
 375         if (original.equals(update)) {
 376             //Reboot use-case
 377             createFibEntries(identifier, update);
 378             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 379                     rd, update.getDestPrefix(), update.getRoutePaths());
 380             return;
 381         }
 382
 383         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 384                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 385     }
 386
 387     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 388         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 389         List<SubTransaction> txnObjects =  new ArrayList<>();
 390         final VpnInstanceOpDataEntry vpnInstance =
 391                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 392         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 393         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 394                 + " has null vpnId!");
 395         final Collection<VpnToDpnList> vpnToDpnList;
 396         if (vrfEntry.getParentVpnRd() != null
 397                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 398             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 399             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 400             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 401                 vpnInstance.getVpnToDpnList();
 402             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 403                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 404         } else {
 405             vpnToDpnList = vpnInstance.getVpnToDpnList();
 406         }
 407         final Uint32 vpnId = vpnInstance.getVpnId();
 408         final String rd = vrfTableKey.getRouteDistinguisher();
 409         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 410         if (subnetRoute != null) {
 411             final long elanTag = subnetRoute.getElantag().toJava();
 412             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 413                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 414             if (vpnToDpnList != null) {
 415                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 416                     () -> Collections.singletonList(
 417                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 418                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 419                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 420                                     installSubnetRouteInFib(curDpn.getDpnId(),
 421                                                                 elanTag, rd, vpnId, vrfEntry, tx);
 422                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
 423                                         vpnId, vrfEntry, NwConstants.ADD_FLOW, tx);
 424                                 }
 425                             }
 426                         })));
 427             }
 428             return;
 429         }
 430         // Get etherType value based on the IpPrefix address family type
 431         int etherType;
 432         try {
 433             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 434         } catch (IllegalArgumentException ex) {
 435             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 436             return;
 437         }
 438
 439         final List<Uint64> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(),
 440                                                                         rd, vrfEntry, etherType);
 441         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 442             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 443                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 444                     final ReentrantLock lock = lockFor(vpnInstance);
 445                     lock.lock();
 446                     try {
 447                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 448                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 449                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 450                                     try {
 451                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 452                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 453                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 454                                                     TransactionAdapter.toWriteTransaction(tx),
 455                                                     txnObjects);
 456                                         } else {
 457                                             createRemoteFibEntry(vpnDpn.getDpnId(),
 458                                                     vpnInstance.getVpnId(),
 459                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 460                                         }
 461                                     } catch (NullPointerException e) {
 462                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 463                                                 vrfEntry.getDestPrefix(), e);
 464                                     }
 465                                 }
 466                             }
 467                         }
 468                     } finally {
 469                         lock.unlock();
 470                     }
 471                 })), MAX_RETRIES);
 472         }
 473
 474         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 475         if (optVpnUuid.isPresent()) {
 476             String vpnUuid = optVpnUuid.get();
 477             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 478             if (interVpnLink != null) {
 479                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 480                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 481                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 482                         // This is an static route that points to the other endpoint of an InterVpnLink
 483                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 484                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 485                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 486                     }
 487                 });
 488             }
 489         }
 490     }
 491
 492     void refreshFibTables(String rd, String prefix) {
 493         InstanceIdentifier<VrfEntry> vrfEntryId =
 494                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 495                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 496         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 497         if (vrfEntry.isPresent()) {
 498             createFibEntries(vrfEntryId, vrfEntry.get());
 499         }
 500     }
 501
 502     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 503         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 504         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 505         prefixBuilder.setDpnId(lri.getDpnId());
 506         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 507         prefixBuilder.setIpAddress(lri.getPrefix());
 508         // Increment the refCount here
 509         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 510             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 511         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 512         if (!isPresentInList) {
 513             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 514             List<String> vpnInstanceNames =
 515                 lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 516             vpnInstanceNames.add(vpnInstanceName);
 517             builder.setVpnInstanceList(vpnInstanceNames);
 518             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 519         } else {
 520             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 521         }
 522         return prefixBuilder.build();
 523     }
 524
 525     void installSubnetRouteInFib(final Uint64 dpnId, final long elanTag, final String rd,
 526             final Uint32 vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 527         if (tx == null) {
 528             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 529                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 530                 "Error installing subnet route in FIB");
 531             return;
 532         }
 533         int etherType;
 534         try {
 535             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 536         } catch (IllegalArgumentException ex) {
 537             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 538             return;
 539         }
 540         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 541             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 542             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 543             final ReentrantLock lock = lockFor(lriKey);
 544             lock.lock();
 545             try {
 546                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 547                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 548
 549                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 550                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 551                                 fibUtil.getVpnInstanceOpData(rd);
 552                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 553                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 554                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 555                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 556                             }
 557                         }
 558                     }
 559                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 560                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 561                 }
 562             } finally {
 563                 lock.unlock();
 564             }
 565         });
 566         final List<InstructionInfo> instructions = new ArrayList<>();
 567         Uint64 subnetRouteMeta = Uint64.valueOf(BigInteger.valueOf(elanTag).shiftLeft(24)
 568             .or(BigInteger.valueOf(vpnId.longValue()).shiftLeft(1)));
 569         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 570         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 571         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 572                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 573         if (vrfEntry.getRoutePaths() != null) {
 574             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 575                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 576                     List<ActionInfo> actionsInfos = new ArrayList<>();
 577                     // reinitialize instructions list for LFIB Table
 578                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 579                     actionsInfos.add(new ActionPopMpls(etherType));
 580                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 581                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 582                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 583                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 584
 585                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions,
 586                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 587                 }
 588             }
 589         }
 590     }
 591
 592     private void installSubnetBroadcastAddrDropRule(final Uint64 dpnId, final String rd, final Uint32 vpnId,
 593             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 594         List<MatchInfo> matches = new ArrayList<>();
 595
 596         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 597                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 598         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 599         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 600         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 601
 602         InetAddress destPrefix;
 603         try {
 604             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 605         } catch (UnknownHostException e) {
 606             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 607                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 608             return;
 609         }
 610
 611         // Match on VpnId and SubnetBroadCast IP address
 612         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 613             MetaDataUtil.METADATA_MASK_VRFID));
 614         matches.add(MatchEthernetType.IPV4);
 615
 616         if (prefixLength != 0) {
 617             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 618         }
 619
 620         //Action is to drop the packet
 621         List<InstructionInfo> dropInstructions = new ArrayList<>();
 622         List<ActionInfo> actionsInfos = new ArrayList<>();
 623         actionsInfos.add(new ActionDrop());
 624         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 625
 626         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 627         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 628         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 629                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 630
 631         Flow flow = flowEntity.getFlowBuilder().build();
 632         String flowId = flowEntity.getFlowId();
 633         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 634         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 635
 636         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 637                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 638                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 639
 640         if (addOrRemove == NwConstants.ADD_FLOW) {
 641             tx.put(flowInstanceId,flow, true);
 642         } else {
 643             tx.delete(flowInstanceId);
 644         }
 645     }
 646
 647     /*
 648      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 649      * LportDispatcher table (via table 80)
 650      */
 651     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 652                                             final VrfEntry vrfEntry, int etherType) {
 653         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 654         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 655         // packet is commuted from Vpn2 to Vpn1.
 656         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 657         if (!interVpnLink.isActive()) {
 658             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 659                      interVpnLinkName, vrfEntry.getDestPrefix());
 660             return;
 661         }
 662
 663         Optional<Uint32> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 664         if (!optLportTag.isPresent()) {
 665             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 666             return;
 667         }
 668
 669         Long lportTag = optLportTag.get().toJava();
 670         Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 671         if (label == null) {
 672             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 673                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 674             return;
 675         }
 676         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 677         List<InstructionInfo> instructions = Arrays.asList(
 678             new InstructionApplyActions(actionsInfos),
 679             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 680                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 681                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 682                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 683             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 684         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 685         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 686
 687         for (Uint64 dpId : targetDpns) {
 688             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 689                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 690
 691             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 692                                /*writeTx*/null);
 693         }
 694     }
 695
 696
 697     /*
 698      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 699      */
 700     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 701                                                final VrfEntry vrfEntry, Uint32 vpnTag) {
 702         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 703         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 704             && vrfEntry.getRoutePaths().size() == 1);
 705         String destination = vrfEntry.getDestPrefix();
 706         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 707         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 708
 709         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 710         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 711         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 712         if (interVpnLink.getState().or(State.Error) != State.Active) {
 713             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 714                 destination, nextHop, interVpnLinkName);
 715             return;
 716         }
 717
 718         Optional<Uint32> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 719         if (!optOtherEndpointLportTag.isPresent()) {
 720             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 721                 vpnUuid, interVpnLinkName);
 722             return;
 723         }
 724
 725         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 726         if (targetDpns.isEmpty()) {
 727             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 728                 vpnUuid, interVpnLinkName);
 729             return;
 730         }
 731
 732         String[] values = destination.split("/");
 733         String destPrefixIpAddress = values[0];
 734         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 735
 736         List<MatchInfo> matches = new ArrayList<>();
 737         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag.longValue()),
 738             MetaDataUtil.METADATA_MASK_VRFID));
 739         matches.add(MatchEthernetType.IPV4);
 740
 741         if (prefixLength != 0) {
 742             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 743         }
 744
 745         List<Instruction> instructions =
 746             Arrays.asList(new InstructionWriteMetadata(
 747                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 748                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 749                             .L3VPN_SERVICE_INDEX)),
 750                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 751                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 752
 753         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 754         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 755         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 756             COOKIE_VM_FIB_TABLE, matches, instructions);
 757
 758         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 759             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 760
 761         for (Uint64 dpId : targetDpns) {
 762
 763             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 764                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 765                 dpId, interVpnLink.getInterVpnLinkName());
 766
 767             mdsalManager.installFlow(dpId, flowEntity);
 768         }
 769     }
 770
 771     private List<Uint64> createLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 772         List<Uint64> returnLocalDpnId = new ArrayList<>();
 773         String localNextHopIP = vrfEntry.getDestPrefix();
 774         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 775         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 776         if (localNextHopInfo == null) {
 777             boolean localNextHopSeen = false;
 778             List<Routes> vpnExtraRoutes = null;
 779             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 780             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 781             // FIXME: use an Identifier structure?
 782             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 783             lock.lock();
 784             try {
 785                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 786                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 787                         vpnName, usedRds, localNextHopIP);
 788                 if (LOG.isDebugEnabled()) {
 789                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 790                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 791                 }
 792
 793                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 794                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 795                     String ipPrefix;
 796                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 797                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 798                     } else {
 799                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 800                     }
 801                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 802                             ipPrefix);
 803                     if (localNextHopInfoLocal != null) {
 804                         localNextHopSeen = true;
 805                         Uint64 dpnId =
 806                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 807                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 808                         returnLocalDpnId.add(dpnId);
 809                     }
 810                 }
 811             } finally {
 812                 lock.unlock();
 813             }
 814             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 815                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 816                 if (optionalLabel.isPresent()) {
 817                     Uint32 label = optionalLabel.get();
 818                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 819                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 820                     final ReentrantLock labelLock = lockFor(lriKey);
 821                     labelLock.lock();
 822                     try {
 823                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 824                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 825                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 826                                     fibUtil.getVpnInstanceOpData(rd);
 827                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 828                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 829                                 if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains(
 830                                        vpnInstanceName)) {
 831                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 832                                     localNextHopIP = lri.getPrefix();
 833                                 } else {
 834                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 835                                     localNextHopIP = lri.getPrefix();
 836                                 }
 837                             }
 838                             if (localNextHopInfo != null) {
 839                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 840                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 841                                 if (vpnExtraRoutes.isEmpty()) {
 842                                     Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 843                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 844                                     returnLocalDpnId.add(dpnId);
 845                                 } else {
 846                                     for (Routes extraRoutes : vpnExtraRoutes) {
 847                                         Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 848                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 849                                         returnLocalDpnId.add(dpnId);
 850                                     }
 851                                 }
 852                             }
 853                         }
 854                     } finally {
 855                         labelLock.unlock();
 856                     }
 857                 }
 858             }
 859             if (returnLocalDpnId.isEmpty()) {
 860                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 861             }
 862         } else {
 863             Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 864                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 865             if (dpnId != null) {
 866                 returnLocalDpnId.add(dpnId);
 867             }
 868         }
 869         return returnLocalDpnId;
 870     }
 871
 872     private Uint64 checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 873                                                 final Uint32 vpnId, final String rd,
 874                                                 final VrfEntry vrfEntry,
 875                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 876                                                 int etherType) {
 877         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 878         if (localNextHopInfo != null) {
 879             long groupId;
 880             long localGroupId;
 881             final Uint64 dpnId = localNextHopInfo.getDpnId();
 882             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 883                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 884                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 885                 return dpnId;
 886             }
 887             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 888                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 889                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 890                 return dpnId;
 891             }
 892             if (!isVpnPresentInDpn(rd, dpnId)) {
 893                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 894                         vpnId, rd, dpnId.toString());
 895                 return Uint64.ZERO;
 896             }
 897             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 898             String prefix = vrfEntry.getDestPrefix();
 899             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 900             //The loadbalancing group is created only if the extra route has multiple nexthops
 901             //to avoid loadbalancing the discovered routes
 902             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 903                     && routes != null) {
 904                 if (vpnExtraRoutes.size() > 1) {
 905                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 906                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 907                 } else {
 908                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 909                     localGroupId = groupId;
 910                 }
 911             } else {
 912                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 913                         gwMacAddress);
 914                 localGroupId = groupId;
 915             }
 916             if (groupId == FibConstants.INVALID_GROUP_ID) {
 917                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 918                         prefix, rd, interfaceName, dpnId.toString());
 919                 return Uint64.ZERO;
 920             }
 921             final List<InstructionInfo> instructions = Collections.singletonList(
 922                     new InstructionApplyActions(
 923                             Collections.singletonList(new ActionGroup(groupId))));
 924             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 925                     new InstructionApplyActions(
 926                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 927             java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 928             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 929             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 930             jobCoordinator.enqueueJob(jobKey,
 931                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 932                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 933                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 934                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 935                         optLabel.ifPresent(label -> {
 936                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 937                                 LOG.debug(
 938                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 939                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 940                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 941                                         nextHopAddressList);
 942                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 943                                         NwConstants.ADD_FLOW, tx);
 944                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 945                             } else {
 946                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 947                                                 + "route. LFib and Terminating table entries will not be created.",
 948                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 949                             }
 950                         });
 951                     }
 952                 })));
 953             return dpnId;
 954         }
 955         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 956                 vpnName);
 957         return Uint64.ZERO;
 958     }
 959
 960     private boolean isVpnPresentInDpn(String rd, Uint64 dpnId)  {
 961         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 962         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 963         return dpnInVpn.isPresent();
 964     }
 965
 966     @Nullable
 967     private LabelRouteInfo getLabelRouteInfo(Uint32 label) {
 968         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 969     }
 970
 971     @Nullable
 972     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 973         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 974             .child(LabelRouteInfo.class, label).build();
 975         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 976         if (opResult.isPresent()) {
 977             return opResult.get();
 978         }
 979         return null;
 980     }
 981
 982     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 983             @Nullable TypedWriteTransaction<Operational> tx) {
 984         if (lri == null) {
 985             return true;
 986         }
 987
 988         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 989         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 990             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 991
 992         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 993             ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 994         if (vpnInstancesList.contains(vpnInstanceName)) {
 995             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 996             vpnInstancesList.remove(vpnInstanceName);
 997         }
 998         if (vpnInstancesList.isEmpty()) {
 999             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
1000             if (tx != null) {
1001                 tx.delete(lriId);
1002             } else {
1003                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
1004             }
1005             return true;
1006         } else {
1007             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
1008             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
1009             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1010         }
1011         return false;
1012     }
1013
1014     void makeTunnelTableEntry(Uint64 dpId, Uint32 label, long groupId/*String egressInterfaceName*/,
1015                                       TypedWriteTransaction<Configuration> tx) {
1016         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1017
1018         createTerminatingServiceActions(dpId, label, actionsInfos, tx);
1019
1020         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1021             dpId, label, groupId);
1022     }
1023
1024     public void createTerminatingServiceActions(Uint64 destDpId, Uint32 label, List<ActionInfo> actionsInfos,
1025                                                 TypedWriteTransaction<Configuration> tx) {
1026         List<MatchInfo> mkMatches = new ArrayList<>();
1027
1028         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1029             destDpId, label, actionsInfos);
1030
1031         // Matching metadata
1032         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1033         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1034
1035         List<InstructionInfo> mkInstructions = new ArrayList<>();
1036         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1037
1038         FlowEntity terminatingServiceTableFlowEntity =
1039             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1040             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1041                     FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1042                     String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1043                     Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()),
1044                     mkMatches, mkInstructions);
1045
1046         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1047
1048         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1049
1050         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1051         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1052             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1053             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1054             .child(Flow.class, flowKey).build();
1055         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1056     }
1057
1058     private void removeTunnelTableEntry(Uint64 dpId, Uint32 label, TypedWriteTransaction<Configuration> tx) {
1059         FlowEntity flowEntity;
1060         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1061         List<MatchInfo> mkMatches = new ArrayList<>();
1062         // Matching metadata
1063         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1064         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1065             NwConstants.INTERNAL_TUNNEL_TABLE,
1066             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1067                 FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1068                 String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1069                 Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()), mkMatches, null);
1070         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1071         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1072         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1073             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1074             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1075
1076         tx.delete(flowInstanceId);
1077         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1078     }
1079
1080     public List<Uint64> deleteLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry) {
1081         List<Uint64> returnLocalDpnId = new ArrayList<>();
1082         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1083         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1084         boolean shouldUpdateNonEcmpLocalNextHop = true;
1085         if (localNextHopInfo == null) {
1086             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1087             if (usedRds.size() > 1) {
1088                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1089                         vrfEntry.getDestPrefix(), vpnName, rd);
1090                 return returnLocalDpnId;
1091             }
1092             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1093             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1094             //in the vpn
1095             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1096                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1097             if (extraRouteOptional.isPresent()) {
1098                 Routes extraRoute = extraRouteOptional.get();
1099                 String ipPrefix;
1100                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1101                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1102                 } else {
1103                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1104                 }
1105                 if (extraRoute.getNexthopIpList().size() > 1) {
1106                     shouldUpdateNonEcmpLocalNextHop = false;
1107                 }
1108                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1109                 if (localNextHopInfo != null) {
1110                     String localNextHopIP = localNextHopInfo.getIpAddress();
1111                     Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1112                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1113                     if (!dpnId.equals(Uint64.ZERO)) {
1114                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1115                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1116                         returnLocalDpnId.add(dpnId);
1117                     }
1118                 } else {
1119                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1120                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1121                 }
1122             }
1123
1124             if (localNextHopInfo == null) {
1125                 /* Imported VRF entry */
1126                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1127                 if (optionalLabel.isPresent()) {
1128                     Uint32 label = optionalLabel.get();
1129                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1130                     LabelRouteInfo lri = getLabelRouteInfo(label);
1131                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1132                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1133                         prefixBuilder.setDpnId(lri.getDpnId());
1134                         Uint64 dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1135                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1136                         if (!dpnId.equals(Uint64.ZERO)) {
1137                             returnLocalDpnId.add(dpnId);
1138                         }
1139                     }
1140                 }
1141             }
1142
1143         } else {
1144             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1145             String localNextHopIP = localNextHopInfo.getIpAddress();
1146             Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1147                     shouldUpdateNonEcmpLocalNextHop);
1148             if (!dpnId.equals(Uint64.ZERO)) {
1149                 returnLocalDpnId.add(dpnId);
1150             }
1151         }
1152
1153         return returnLocalDpnId;
1154     }
1155
1156     private Uint64 checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1157             final String vpnName, final Uint32 vpnId, final String rd, final VrfEntry vrfEntry,
1158             boolean shouldUpdateNonEcmpLocalNextHop) {
1159         if (localNextHopInfo != null) {
1160             final Uint64 dpnId = localNextHopInfo.getDpnId();
1161             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1162                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1163                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1164                 return dpnId;
1165             }
1166             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1167                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1168                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1169                 return dpnId;
1170             }
1171
1172             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1173                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1174                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1175                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1176                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1177                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1178                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1179                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1180                                         NwConstants.DEL_FLOW, tx);
1181                                 removeTunnelTableEntry(dpnId, label, tx);
1182                             });
1183                         }
1184                     }
1185                 })));
1186             //TODO: verify below adjacency call need to be optimized (?)
1187             //In case of the removal of the extra route, the loadbalancing group is updated
1188             if (shouldUpdateNonEcmpLocalNextHop) {
1189                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1190             }
1191             return dpnId;
1192         }
1193         return Uint64.ZERO;
1194     }
1195
1196     private void createRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1197             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1198         if (tx == null) {
1199             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1200                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1201                 "Error creating remote FIB entry");
1202             return;
1203         }
1204
1205         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1206         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", vrfEntry.getDestPrefix(), rd,
1207                 remoteDpnId);
1208
1209         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1210             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1211             return;
1212         }
1213         // Handling static VRF entries
1214         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1215         List<Routes> vpnExtraRoutes =
1216                 VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, vpnName, usedRds, vrfEntry.getDestPrefix());
1217         if (!vpnExtraRoutes.isEmpty()) {
1218             programRemoteFibWithLoadBalancingGroups(remoteDpnId, vpnId, rd, vrfEntry, vpnExtraRoutes);
1219         } else {
1220             // Program in case of other static VRF entries like floating IPs
1221             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1222         }
1223     }
1224
1225     // Allow deprecated TransactionRunner calls for now
1226     @SuppressWarnings("ForbidCertainMethod")
1227     private void programRemoteFibWithLoadBalancingGroups(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1228             final VrfEntry vrfEntry, List<Routes> vpnExtraRoutes) {
1229         // create loadbalancing groups for extra routes only when the extra route is
1230         // present behind multiple VMs
1231         // Obtain the local routes for this particular dpn.
1232         java.util.Optional<Routes> routes = vpnExtraRoutes.stream().filter(route -> {
1233             Prefixes prefixToInterface =
1234                     fibUtil.getPrefixToInterface(vpnId, FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1235             if (prefixToInterface == null) {
1236                 return false;
1237             }
1238             return remoteDpnId.equals(prefixToInterface.getDpnId());
1239         }).findFirst();
1240         long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1241                 routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1242         if (groupId == FibConstants.INVALID_GROUP_ID) {
1243             LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", vrfEntry.getDestPrefix(), rd,
1244                     remoteDpnId);
1245             return;
1246         }
1247         List<ActionInfo> actionInfos = Collections.singletonList(new ActionGroup(groupId));
1248         List<InstructionInfo> instructions = Lists.newArrayList(new InstructionApplyActions(actionInfos));
1249         String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1250         jobCoordinator.enqueueJob(jobKey,
1251             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1252                 baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1253                         NwConstants.ADD_FLOW, txn, null);
1254             })));
1255
1256         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1257     }
1258
1259     private void programRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1260             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1261         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1262         if (adjacencyResults.isEmpty()) {
1263             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", vrfEntry.getRoutePaths(), rd,
1264                     remoteDpnId);
1265             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1266             return;
1267         }
1268         baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, TransactionAdapter.toWriteTransaction(tx),
1269                 rd, adjacencyResults, null);
1270         LOG.debug("Successfully programmed FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1271     }
1272
1273     protected void cleanUpOpDataForFib(Uint32 vpnId, String primaryRd, final VrfEntry vrfEntry) {
1274     /* Get interface info from prefix to interface mapping;
1275         Use the interface info to get the corresponding vpn interface op DS entry,
1276         remove the adjacency corresponding to this fib entry.
1277         If adjacency removed is the last adjacency, clean up the following:
1278          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1279          - prefix to interface entry
1280          - vpn interface op DS
1281      */
1282         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1283         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1284         if (prefixInfo == null) {
1285             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1286             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1287             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1288             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
1289                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1290                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1291                     if (nextHopIp != null) {
1292                         String ipPrefix;
1293                         if (isIpv4Address(nextHopIp)) {
1294                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1295                         } else {
1296                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1297                         }
1298                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1299                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1300                     }
1301                 }
1302             }
1303             if (prefixInfo == null) {
1304                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1305                 if (optionalLabel.isPresent()) {
1306                     Uint32 label = optionalLabel.get();
1307                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1308                     LabelRouteInfo lri = getLabelRouteInfo(label);
1309                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1310                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1311                         prefixBuilder.setDpnId(lri.getDpnId());
1312                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1313                         prefixBuilder.setIpAddress(lri.getPrefix());
1314                         prefixInfo = prefixBuilder.build();
1315                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1316                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1317                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1318                     }
1319                 }
1320             }
1321         } else {
1322             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1323         }
1324     }
1325
1326     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1327                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1328
1329         if (prefixInfo == null) {
1330             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1331                     vrfEntry.getDestPrefix(), vpnId, rd);
1332             return; //Don't have any info for this prefix (shouldn't happen); need to return
1333         }
1334
1335         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1336             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1337                     vrfEntry.getDestPrefix(), vpnId, rd);
1338             return;
1339         }
1340
1341         String ifName = prefixInfo.getVpnInterfaceName();
1342         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1343             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1344     }
1345
1346     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1347         Prefixes prefixInfo;
1348         Uint32 vpnId;
1349         String rd;
1350         VrfEntry vrfEntry;
1351         Routes extraRoute;
1352
1353         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1354                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1355             this.prefixInfo = prefixInfo;
1356             this.vpnId = vpnId;
1357             this.rd = rd;
1358             this.vrfEntry = vrfEntry;
1359             this.extraRoute = extraRoute;
1360         }
1361
1362         @Override
1363         public List<ListenableFuture<Void>> call() {
1364             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1365             // to call the respective helpers.
1366             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1367                 //First Cleanup LabelRouteInfo
1368                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1369                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1370                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1371                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1372                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1373                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1374                         final ReentrantLock lock = lockFor(lriKey);
1375                         lock.lock();
1376                         try {
1377                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1378                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1379                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1380                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1381                                         fibUtil.getVpnInstanceOpData(rd);
1382                                 String vpnInstanceName = "";
1383                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1384                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1385                                 }
1386                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1387                                 if (lriRemoved) {
1388                                     String parentRd = lri.getParentVpnRd();
1389                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1390                                             parentRd, vrfEntry.getDestPrefix()));
1391                                 }
1392                             } else {
1393                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1394                                         rd, vrfEntry.getDestPrefix()));
1395                             }
1396                         } finally {
1397                             lock.unlock();
1398                         }
1399                     });
1400                 }
1401                 String ifName = prefixInfo.getVpnInterfaceName();
1402                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1403                 String vpnName = null;
1404
1405                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1406                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1407                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1408                     if (vpnNameOpt.isPresent()) {
1409                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1410                     }
1411                 }
1412                 if (optVpnName.isPresent()) {
1413                     vpnName = optVpnName.get();
1414                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1415                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1416                     if (opVpnInterface.isPresent()) {
1417                         Uint32 associatedVpnId = fibUtil.getVpnId(vpnName);
1418                         if (Objects.equals(vpnId, associatedVpnId)) {
1419                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1420                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1421                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1422                             return;
1423                         } else {
1424                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1425                                     vrfEntry.getDestPrefix(), associatedVpnId);
1426                         }
1427                     }
1428                 }
1429                 if (extraRoute != null) {
1430                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1431                     //Only one used Rd present in case of removal event
1432                     String usedRd = usedRds.get(0);
1433                     if (optVpnName.isPresent()) {
1434                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1435                                         vrfEntry.getDestPrefix()));
1436                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1437                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1438                     }
1439                 }
1440                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1441             }));
1442         }
1443     }
1444
1445     /**
1446      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1447      * Remove Adjacency from VPNInterfaceOpData.
1448      * if Adjacency != primary.
1449      * if Adjacency == primary , then mark it for deletion.
1450      * Remove entire VPNinterfaceOpData Entry.
1451      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1452      * @param vrfEntry - VrfEntry removed
1453      * @param ifName - Interface name from VRFentry
1454      * @param vpnName - VPN name of corresponding VRF
1455      * @param tx - ReadWrite Tx
1456      */
1457     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
1458             justification = "https://github.com/spotbugs/spotbugs/issues/811")
1459     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1460                                                           TypedReadWriteTransaction<Operational> tx)
1461             throws ExecutionException, InterruptedException {
1462         InstanceIdentifier<Adjacency> adjacencyIid =
1463                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1464         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1465         if (adjacencyOptional.isPresent()) {
1466             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1467                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1468             } else {
1469                 tx.merge(adjacencyIid,
1470                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1471             }
1472         }
1473
1474         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1475
1476         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1477             return;
1478         }
1479
1480         @NonNull List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
1481         if (adjacencies.size() <= 2
1482                 && adjacencies.stream().allMatch(adjacency ->
1483                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1484                         && adjacency.isMarkedForDeletion() != null
1485                         && adjacency.isMarkedForDeletion()
1486         )) {
1487             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1488             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1489         }
1490     }
1491
1492     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1493         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1494         final String rd = vrfTableKey.getRouteDistinguisher();
1495         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1496         if (vpnInstance == null) {
1497             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1498             return;
1499         }
1500         final Collection<VpnToDpnList> vpnToDpnList;
1501         if (vrfEntry.getParentVpnRd() != null
1502                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1503             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1504             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1505             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1506                     vpnInstance.getVpnToDpnList();
1507             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1508                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1509         } else {
1510             vpnToDpnList = vpnInstance.getVpnToDpnList();
1511         }
1512
1513         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1514         final java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1515         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1516         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1517         if (subnetRoute != null) {
1518             long elanTag = subnetRoute.getElantag().toJava();
1519             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1520                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1521             if (vpnToDpnList != null) {
1522                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1523                     () -> Collections.singletonList(
1524                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1525                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1526
1527                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(),
1528                                     vpnInstance.getVpnId(),
1529                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1530                                     TransactionAdapter.toWriteTransaction(tx), null);
1531                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1532                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(),
1533                                         label, null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1534                                 }
1535
1536                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
1537                                     vpnInstance.getVpnId(),
1538                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1539                             }
1540                         })));
1541             }
1542             optionalLabel.ifPresent(label -> {
1543                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1544                 final ReentrantLock lock = lockFor(lriKey);
1545                 lock.lock();
1546                 try {
1547                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1548                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1549                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1550                                 fibUtil.getVpnInstanceOpData(rd);
1551                         String vpnInstanceName = "";
1552                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1553                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1554                         }
1555                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1556                         if (lriRemoved) {
1557                             String parentRd = lri.getParentVpnRd();
1558                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1559                                     parentRd, vrfEntry.getDestPrefix()));
1560                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1561                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1562                         }
1563                     } else {
1564                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1565                                 rd, vrfEntry.getDestPrefix()));
1566                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1567                                 label, rd, vrfEntry.getDestPrefix());
1568                     }
1569                 } finally {
1570                     lock.unlock();
1571                 }
1572             });
1573             return;
1574         }
1575
1576         final List<Uint64> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1577             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1578         if (vpnToDpnList != null) {
1579             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1580                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1581             String jobKey;
1582             Optional<Routes> extraRouteOptional;
1583             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1584             if (usedRds != null && !usedRds.isEmpty()) {
1585                 if (usedRds.size() > 1) {
1586                     LOG.error("The extra route prefix is still present in some DPNs");
1587                     return ;
1588                 } else {
1589                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1590                     //is not present in any other DPN
1591                     extraRouteOptional = VpnExtraRouteHelper
1592                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1593                 }
1594             } else {
1595                 extraRouteOptional = Optional.absent();
1596             }
1597
1598             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1599                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1600                     if (localDpnIdList.size() <= 0) {
1601                         for (VpnToDpnList curDpn : vpnToDpnList) {
1602                             baseVrfEntryHandler.deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(),
1603                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1604                                 TransactionAdapter.toWriteTransaction(tx));
1605                         }
1606                     } else {
1607                         for (Uint64 localDpnId : localDpnIdList) {
1608                             for (VpnToDpnList curDpn : vpnToDpnList) {
1609                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1610                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1611                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1612                                         TransactionAdapter.toWriteTransaction(tx));
1613                                 }
1614                             }
1615                         }
1616                     }
1617                     if (extraRouteOptional.isPresent()) {
1618                         //Remove select groups only for extra-routes
1619                         nextHopManager.removeNextHopPointer(nextHopManager
1620                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1621                         nextHopManager.removeNextHopPointer(nextHopManager
1622                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1623                     }
1624                 })), MAX_RETRIES);
1625         }
1626
1627         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1628         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1629         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1630
1631         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1632         // of the interVpnLink.
1633         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1634         if (optVpnUuid.isPresent()) {
1635             String vpnUuid = optVpnUuid.get();
1636             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1637                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1638                 if (optInterVpnLink.isPresent()) {
1639                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1640                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1641                         // This is route that points to the other endpoint of an InterVpnLink
1642                         // In that case, we should look for the FIB table pointing to
1643                         // LPortDispatcher table and remove it.
1644                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1645                     }
1646                 }
1647             });
1648         }
1649
1650     }
1651
1652     private void makeLFibTableEntry(Uint64 dpId, Uint32 label, @Nullable List<InstructionInfo> instructions,
1653                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1654         if (tx == null) {
1655             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1656                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1657                 "Error making LFIB table entry");
1658             return;
1659         }
1660
1661         List<MatchInfo> matches = new ArrayList<>();
1662         matches.add(MatchEthernetType.MPLS_UNICAST);
1663         matches.add(new MatchMplsLabel(label.longValue()));
1664
1665         // Install the flow entry in L3_LFIB_TABLE
1666         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1667
1668         FlowEntity flowEntity;
1669         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1670             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1671         Flow flow = flowEntity.getFlowBuilder().build();
1672         String flowId = flowEntity.getFlowId();
1673         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1674         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1675         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1676             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1677             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1678
1679         if (addOrRemove == NwConstants.ADD_FLOW) {
1680             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1681         } else {
1682             tx.delete(flowInstanceId);
1683         }
1684
1685         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1686             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1687     }
1688
1689     public void populateFibOnNewDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1690                                     final FutureCallback<List<Void>> callback) {
1691         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1692         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1693             () -> {
1694                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1695                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1696                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1697                         LogicalDatastoreType.CONFIGURATION, id);
1698                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1699                 if (!vrfTable.isPresent()) {
1700                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1701                     if (callback != null) {
1702                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1703                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1704                     }
1705                     return futures;
1706                 }
1707
1708                 final ReentrantLock lock = lockFor(vpnInstance);
1709                 lock.lock();
1710                 try {
1711                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1712                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1713                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1714                             if (subnetRoute != null) {
1715                                 long elanTag = subnetRoute.getElantag().toJava();
1716                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1717                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1718                                         tx);
1719                                 continue;
1720                             }
1721                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1722                             if (routerInt != null) {
1723                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1724                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1725                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1726                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1727                                         NwConstants.ADD_FLOW);
1728                                 continue;
1729                             }
1730                             //Handle local flow creation for imports
1731                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1732                                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1733                                 if (optionalLabel.isPresent()) {
1734                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1735                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1736                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1737                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1738                                             try {
1739                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1740                                                         vrfEntry.getDestPrefix());
1741                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1742                                             } catch (IllegalArgumentException ex) {
1743                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1744                                                         vrfEntry.getDestPrefix());
1745                                             }
1746                                             continue;
1747                                         }
1748                                     }
1749                                 }
1750                             }
1751                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1752                                     vrfEntry, dpnId);
1753                             if (shouldCreateRemoteFibEntry) {
1754                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1755                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1756                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1757                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1758                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1759                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1760                                 } else {
1761                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1762                                             vrfEntry, tx);
1763                                 }
1764                             }
1765                         }
1766                     }));
1767                     if (callback != null) {
1768                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1769                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1770                     }
1771                 } finally {
1772                     lock.unlock();
1773                 }
1774                 return futures;
1775             });
1776     }
1777
1778     public void populateExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1779                                             final String localNextHopIp, final String remoteNextHopIp) {
1780         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1781             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1782         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1783         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1784         List<SubTransaction> txnObjects =  new ArrayList<>();
1785         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1786         if (vrfTable.isPresent()) {
1787             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1788                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1789                     final ReentrantLock lock = lockFor(vpnInstance);
1790                     lock.lock();
1791                     try {
1792                         vrfTable.get().nonnullVrfEntry().stream()
1793                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1794                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1795                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1796                     } finally {
1797                         lock.unlock();
1798                     }
1799                 })));
1800         }
1801     }
1802
1803     public void manageRemoteRouteOnDPN(final boolean action,
1804                                        final Uint64 localDpnId,
1805                                        final Uint32 vpnId,
1806                                        final String rd,
1807                                        final String destPrefix,
1808                                        final String destTepIp,
1809                                        final Uint32 label) {
1810         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1811
1812         if (vpnInstance == null) {
1813             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1814             return;
1815         }
1816
1817         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1818             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1819                 final ReentrantLock lock = lockFor(vpnInstance);
1820                 lock.lock();
1821                 try {
1822                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1823                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1824                     if (vrfEntry == null) {
1825                         return;
1826                     }
1827                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1828                             action, localDpnId, vpnId, rd, destPrefix);
1829                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1830                     VrfEntry modVrfEntry;
1831                     if (routePathList == null || routePathList.isEmpty()) {
1832                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1833                                 Collections.singletonList(destTepIp),
1834                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1835                     } else {
1836                         modVrfEntry = vrfEntry;
1837                     }
1838
1839                     if (action) {
1840                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1841                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1842                                 modVrfEntry, tx);
1843                     } else {
1844                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1845                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1846                                 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1847                         if (usedRds.size() > 1) {
1848                             LOG.debug("The extra route prefix is still present in some DPNs");
1849                             return;
1850                         }
1851                         //Is this fib route an extra route? If yes, get the nexthop which would be
1852                         //an adjacency in the vpn
1853                         Optional<Routes> extraRouteOptional = Optional.absent();
1854                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC && usedRds.size() != 0) {
1855                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1856                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1857                                     usedRds.get(0), vrfEntry.getDestPrefix());
1858                         }
1859                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1860                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1861                     }
1862                 } finally {
1863                     lock.unlock();
1864                 }
1865             })));
1866     }
1867
1868     public void cleanUpDpnForVpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1869                                  final FutureCallback<List<Void>> callback) {
1870         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1871         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1872             () -> {
1873                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1874                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1875                 List<SubTransaction> txnObjects =  new ArrayList<>();
1876                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1877                         LogicalDatastoreType.CONFIGURATION, id);
1878                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1879                 if (vrfTable.isPresent()) {
1880                     final ReentrantLock lock = lockFor(vpnInstance);
1881                     lock.lock();
1882                     try {
1883                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1884                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1885                             for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1886                                 /* parentRd is only filled for external PNF cases where the interface on the external
1887                                  * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1888                                  * #fibUtil.isInterfacePresentInDpn().
1889                                 * */
1890                                 String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1891                                         : rd;
1892                                 /* Handle subnet routes here */
1893                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1894                                 if (subnetRoute != null && !fibUtil
1895                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1896                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1897                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1898                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1899                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1900                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1901                                     if (routePaths != null) {
1902                                         for (RoutePaths routePath : routePaths) {
1903                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1904                                                     DEFAULT_FIB_FLOW_PRIORITY,
1905                                                     NwConstants.DEL_FLOW, tx);
1906                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1907                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1908                                                     vrfEntry.getDestPrefix());
1909                                         }
1910                                     }
1911                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1912                                             NwConstants.DEL_FLOW, tx);
1913                                     continue;
1914                                 }
1915                                 // ping responder for router interfaces
1916                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1917                                 if (routerInt != null) {
1918                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1919                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1920                                             routerInt.getMacAddress());
1921                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1922                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1923                                             NwConstants.DEL_FLOW);
1924                                     continue;
1925                                 }
1926
1927                                 //Handle local flow deletion for imports
1928                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1929                                     java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1930                                     if (optionalLabel.isPresent()) {
1931                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1932                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1933                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1934                                                 lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1935                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1936                                         }
1937                                     }
1938                                 }
1939
1940                                 // Passing null as we don't know the dpn
1941                                 // to which prefix is attached at this point
1942                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1943                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1944                                 Optional<Routes> extraRouteOptional;
1945                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1946                                 //an adjacency in the vpn
1947                                 if (usedRds != null && !usedRds.isEmpty()) {
1948                                     if (usedRds.size() > 1) {
1949                                         LOG.error("The extra route prefix is still present in some DPNs");
1950                                         return;
1951                                     } else {
1952                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1953                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1954
1955                                     }
1956                                 } else {
1957                                     extraRouteOptional = Optional.absent();
1958                                 }
1959                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1960                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1961                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1962                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1963                                 } else {
1964                                     if (subnetRoute == null || !fibUtil
1965                                             .isInterfacePresentInDpn(parentRd, dpnId)) {
1966                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1967                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1968                                             TransactionAdapter.toWriteTransaction(tx));
1969                                     }
1970                                 }
1971                             }
1972                         }));
1973                     } finally {
1974                         lock.unlock();
1975                     }
1976                     if (callback != null) {
1977                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1978                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1979                     }
1980                 } else {
1981                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1982                 }
1983                 return futures;
1984             });
1985
1986     }
1987
1988     public void cleanUpExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1989                                            final String localNextHopIp, final String remoteNextHopIp) {
1990         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1991                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1992             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1993         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1994         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1995         List<SubTransaction> txnObjects =  new ArrayList<>();
1996         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1997         if (vrfTable.isPresent()) {
1998             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1999                 () -> {
2000                     final ReentrantLock lock = lockFor(vpnInstance);
2001                     lock.lock();
2002                     try {
2003                         return Collections.singletonList(
2004                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
2005                                 tx -> vrfTable.get().nonnullVrfEntry().stream()
2006                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2007                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
2008                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
2009                                         txnObjects))));
2010                     } finally {
2011                         lock.unlock();
2012                     }
2013                 });
2014         }
2015     }
2016
2017     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2018         InstanceIdentifierBuilder<VrfTables> idBuilder =
2019             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2020         return idBuilder.build();
2021     }
2022
2023     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2024         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2025                 .FLOWID_SEPARATOR + nextHop;
2026     }
2027
2028     private String getTableMissFlowRef(Uint64 dpnId, short tableId, Uint32 tableMiss) {
2029         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2030                 + tableMiss + FLOWID_PREFIX;
2031     }
2032
2033     @Nullable
2034     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2035         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2036             .child(VrfTables.class, new VrfTablesKey(rd))
2037             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2038         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2039         if (vrfEntry.isPresent()) {
2040             return vrfEntry.get();
2041         }
2042         return null;
2043     }
2044
2045     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2046                                              final String vpnName,
2047                                              final VrfEntry vrfEntry) {
2048         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2049
2050         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2051         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2052
2053         if (targetDpns.isEmpty()) {
2054             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2055             return;
2056         }
2057
2058         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2059         java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2060
2061         // delete from FIB
2062         //
2063         optNextHop.ifPresent(nextHop -> {
2064             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2065             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2066             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2067                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2068
2069             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2070             for (Uint64 dpId : targetDpns) {
2071                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2072                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2073
2074                 mdsalManager.removeFlow(dpId, flow);
2075             }
2076         });
2077
2078         // delete from LFIB
2079         //
2080         optLabel.ifPresent(label -> {
2081             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2082
2083             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2084                 for (Uint64 dpId : targetDpns) {
2085                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2086                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2087                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY,
2088                             NwConstants.DEL_FLOW, tx);
2089                 }
2090             }), LOG, "Error removing flows");
2091         });
2092     }
2093
2094     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2095             List<String> nextHopAddressList, LabelRouteInfo lri) {
2096         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2097                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2098     }
2099
2100     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Uint32 vpnId, VrfEntry vrfEntry, Uint64 dpnId) {
2101         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2102             return true;
2103         }
2104
2105         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2106         if (prefix != null) {
2107             Uint64 prefixDpnId = prefix.getDpnId();
2108             if (dpnId.equals(prefixDpnId)) {
2109                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2110                         vrfEntry, dpnId);
2111                 return false;
2112             }
2113         }
2114         return true;
2115     }
2116
2117     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2118         // FIXME: use vpnInstance.key() instead?
2119         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2120     }
2121
2122     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2123         return JvmGlobalLocks.getLockFor(label);
2124     }
2125 }