fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.net.InetAddress;
  25 import java.net.UnknownHostException;
  26 import java.util.ArrayList;
  27 import java.util.Arrays;
  28 import java.util.Collection;
  29 import java.util.Collections;
  30 import java.util.List;
  31 import java.util.Objects;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.locks.ReentrantLock;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.Datastore.Operational;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.TransactionAdapter;
  51 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  52 import org.opendaylight.genius.infra.TypedWriteTransaction;
  53 import org.opendaylight.genius.mdsalutil.ActionInfo;
  54 import org.opendaylight.genius.mdsalutil.FlowEntity;
  55 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  56 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  57 import org.opendaylight.genius.mdsalutil.MatchInfo;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NWUtil;
  60 import org.opendaylight.genius.mdsalutil.NwConstants;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  73 import org.opendaylight.genius.utils.JvmGlobalLocks;
  74 import org.opendaylight.genius.utils.ServiceIndex;
  75 import org.opendaylight.genius.utils.batching.SubTransaction;
  76 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  77 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  78 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  79 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  80 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  81 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  85 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  86 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127
 128 @Singleton
 129 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 130
 131     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 132     private static final String FLOWID_PREFIX = "L3.";
 133     private static final Uint64 COOKIE_VM_FIB_TABLE =  Uint64.valueOf("8000003", 16).intern();
 134     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 135     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 136     private static final int LFIB_INTERVPN_PRIORITY = 15;
 137     public static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 138     private static final int MAX_RETRIES = 3;
 139     private static final Uint64 COOKIE_TABLE_MISS = Uint64.valueOf("8000004", 16).intern();
 140
 141     private final DataBroker dataBroker;
 142     private final ManagedNewTransactionRunner txRunner;
 143     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 144     private final IMdsalApiManager mdsalManager;
 145     private final NexthopManager nextHopManager;
 146     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 147     private final BaseVrfEntryHandler baseVrfEntryHandler;
 148     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 149     private final JobCoordinator jobCoordinator;
 150     private final IElanService elanManager;
 151     private final FibUtil fibUtil;
 152     private final InterVpnLinkCache interVpnLinkCache;
 153     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 154     private final UpgradeState upgradeState;
 155     private final DataTreeEventCallbackRegistrar eventCallbacks;
 156
 157     @Inject
 158     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 159                             final NexthopManager nexthopManager,
 160                             final IElanService elanManager,
 161                             final BaseVrfEntryHandler vrfEntryHandler,
 162                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 163                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 164                             final JobCoordinator jobCoordinator,
 165                             final FibUtil fibUtil,
 166                             final InterVpnLinkCache interVpnLinkCache,
 167                             final UpgradeState upgradeState,
 168                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 169         super(VrfEntry.class, VrfEntryListener.class);
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 173         this.mdsalManager = mdsalApiManager;
 174         this.nextHopManager = nexthopManager;
 175         this.elanManager = elanManager;
 176         this.baseVrfEntryHandler = vrfEntryHandler;
 177         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 178         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 179         this.jobCoordinator = jobCoordinator;
 180         this.fibUtil = fibUtil;
 181         this.interVpnLinkCache = interVpnLinkCache;
 182         this.upgradeState = upgradeState;
 183         this.eventCallbacks = eventCallbacks;
 184     }
 185
 186     @Override
 187     @PostConstruct
 188     public void init() {
 189         LOG.info("{} init", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     @SuppressWarnings("checkstyle:IllegalCatch")
 195     public void close() {
 196         closeables.forEach(c -> {
 197             try {
 198                 c.close();
 199             } catch (Exception e) {
 200                 LOG.warn("Error closing {}", c, e);
 201             }
 202         });
 203     }
 204
 205     @Override
 206     protected VrfEntryListener getDataTreeChangeListener() {
 207         return VrfEntryListener.this;
 208     }
 209
 210     @Override
 211     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 212         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 213     }
 214
 215     @Override
 216     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 217         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 218         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 219         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 220                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 221         addFibEntries(identifier, vrfEntry, rd);
 222         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 223                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 224     }
 225
 226     //This method is temporary. Eventually Factory design pattern will be used to get
 227     // right VrfEntryhandle and invoke its methods.
 228     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 229         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 230             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 234             LOG.info("EVPN flows need to be programmed.");
 235             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 236                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 237             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 238             closeables.add(evpnVrfEntryHandler);
 239             return;
 240         }
 241         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 242         if (routerInt != null) {
 243             // ping responder for router interfaces
 244             routerInterfaceVrfEntryHandler.createFlows(vrfEntry, rd);
 245             return;
 246         }
 247         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 248             createFibEntries(identifier, vrfEntry);
 249             return;
 250         }
 251     }
 252
 253     @Override
 254     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 255         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 256         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 257         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 258                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 259         removeFibEntries(identifier, vrfEntry, rd);
 260         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 261             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 262     }
 263
 264     //This method is temporary. Eventually Factory design pattern will be used to get
 265     // right VrfEntryhandle and invoke its methods.
 266     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 267         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 268             LOG.info("EVPN flows to be deleted");
 269             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 270                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 271             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 272             closeables.add(evpnVrfEntryHandler);
 273             return;
 274         }
 275         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 276         if (routerInt != null) {
 277             // ping responder for router interfaces
 278             routerInterfaceVrfEntryHandler.removeFlows(vrfEntry, rd);
 279             return;
 280         }
 281         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 282             deleteFibEntries(identifier, vrfEntry);
 283             return;
 284         }
 285         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 286             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 287             return;
 288         }
 289     }
 290
 291     @Override
 292     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 293     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 294     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 295     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 296         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 297         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 298         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 299                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 300         // Handle BGP Routes first
 301         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 302             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 303             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 304                     rd, update.getDestPrefix(), update.getRoutePaths());
 305             return;
 306         }
 307
 308         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 309             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 310             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 311             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 312
 313             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 314             //updated vrf entry route path is null. This can happen during tunnel events.
 315             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 316             List<String> usedRds = new ArrayList<>();
 317             if (optVpnInstance.isPresent()) {
 318                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 319                         update.getDestPrefix());
 320             }
 321             // If original VRF Entry had nexthop null , but update VRF Entry
 322             // has nexthop , route needs to be created on remote Dpns
 323             if (originalRoutePath == null || originalRoutePath.isEmpty()
 324                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 325                 // TODO(vivek): Though ugly, Not handling this code now, as each
 326                 // tep add event will invoke flow addition
 327                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 328                         + " This event is IGNORED here.", update.getDestPrefix());
 329                 return;
 330             }
 331
 332             // If original VRF Entry had valid nexthop , but update VRF Entry
 333             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 334             if (updateRoutePath == null || updateRoutePath.isEmpty()
 335                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 336                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 337                         + "This event is IGNORED here.", update.getDestPrefix());
 338                 return;
 339             }
 340             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 341             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 342             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 343             List<ListenableFuture<Void>> futures = new ArrayList<>();
 344             ListenableFuture<Void> configFuture =
 345                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 346                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 347                         nextHopsRemoved.parallelStream()
 348                             .forEach(nextHopRemoved -> {
 349                                 try {
 350                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 351                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 352                                 } catch (ExecutionException | InterruptedException e) {
 353                                     throw new RuntimeException(e);
 354                                 }
 355                             }))));
 356             futures.add(configFuture);
 357             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 358                 @Override
 359                 public void onSuccess(Void result) {
 360                     createFibEntries(identifier, update);
 361                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 362                             rd, update.getDestPrefix(), update.getRoutePaths());
 363                 }
 364
 365                 @Override
 366                 public void onFailure(Throwable throwable) {
 367                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 368                             update, throwable);
 369                 }
 370             }, MoreExecutors.directExecutor());
 371             return;
 372         }
 373
 374         //Handle all other routes only on a cluster reboot
 375         if (original.equals(update)) {
 376             //Reboot use-case
 377             createFibEntries(identifier, update);
 378             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 379                     rd, update.getDestPrefix(), update.getRoutePaths());
 380             return;
 381         }
 382
 383         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 384                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 385     }
 386
 387     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 388         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 389         List<SubTransaction> txnObjects =  new ArrayList<>();
 390         final VpnInstanceOpDataEntry vpnInstance =
 391                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 392         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 393         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 394                 + " has null vpnId!");
 395         final Collection<VpnToDpnList> vpnToDpnList;
 396         if (vrfEntry.getParentVpnRd() != null
 397                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 398             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 399             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 400             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 401                 vpnInstance.getVpnToDpnList();
 402             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 403                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 404         } else {
 405             vpnToDpnList = vpnInstance.getVpnToDpnList();
 406         }
 407         final Uint32 vpnId = vpnInstance.getVpnId();
 408         final String rd = vrfTableKey.getRouteDistinguisher();
 409         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 410         if (subnetRoute != null) {
 411             final long elanTag = subnetRoute.getElantag().toJava();
 412             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 413                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 414             if (vpnToDpnList != null) {
 415                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 416                     () -> Collections.singletonList(
 417                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 418                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 419                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 420                                     installSubnetRouteInFib(curDpn.getDpnId(),
 421                                                                 elanTag, rd, vpnId, vrfEntry, tx);
 422                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
 423                                         vpnId, vrfEntry, NwConstants.ADD_FLOW, tx);
 424                                 }
 425                             }
 426                         })));
 427             }
 428             return;
 429         }
 430         // Get etherType value based on the IpPrefix address family type
 431         int etherType;
 432         try {
 433             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 434         } catch (IllegalArgumentException ex) {
 435             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 436             return;
 437         }
 438
 439         final List<Uint64> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(),
 440                                                                         rd, vrfEntry, etherType);
 441         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 442             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 443                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 444                     final ReentrantLock lock = lockFor(vpnInstance);
 445                     lock.lock();
 446                     try {
 447                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 448                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 449                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 450                                     try {
 451                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 452                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 453                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 454                                                     TransactionAdapter.toWriteTransaction(tx),
 455                                                     txnObjects);
 456                                         } else {
 457                                             createRemoteFibEntry(vpnDpn.getDpnId(),
 458                                                     vpnInstance.getVpnId(),
 459                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 460                                         }
 461                                     } catch (NullPointerException e) {
 462                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 463                                                 vrfEntry.getDestPrefix(), e);
 464                                     }
 465                                 }
 466                             }
 467                         }
 468                     } finally {
 469                         lock.unlock();
 470                     }
 471                 })), MAX_RETRIES);
 472         }
 473
 474         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 475         if (optVpnUuid.isPresent()) {
 476             String vpnUuid = optVpnUuid.get();
 477             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 478             if (interVpnLink != null) {
 479                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 480                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 481                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 482                         // This is an static route that points to the other endpoint of an InterVpnLink
 483                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 484                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 485                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 486                     }
 487                 });
 488             }
 489         }
 490     }
 491
 492     void refreshFibTables(String rd, String prefix) {
 493         InstanceIdentifier<VrfEntry> vrfEntryId =
 494                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 495                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 496         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 497         if (vrfEntry.isPresent()) {
 498             createFibEntries(vrfEntryId, vrfEntry.get());
 499         }
 500     }
 501
 502     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 503         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 504         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 505         prefixBuilder.setDpnId(lri.getDpnId());
 506         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 507         prefixBuilder.setIpAddress(lri.getPrefix());
 508         // Increment the refCount here
 509         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 510             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 511         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 512         if (!isPresentInList) {
 513             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 514             List<String> vpnInstanceNames =
 515                 lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 516             vpnInstanceNames.add(vpnInstanceName);
 517             builder.setVpnInstanceList(vpnInstanceNames);
 518             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 519         } else {
 520             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 521         }
 522         return prefixBuilder.build();
 523     }
 524
 525     void installSubnetRouteInFib(final Uint64 dpnId, final long elanTag, final String rd,
 526             final Uint32 vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 527         if (tx == null) {
 528             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 529                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 530                 "Error installing subnet route in FIB");
 531             return;
 532         }
 533         int etherType;
 534         try {
 535             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 536         } catch (IllegalArgumentException ex) {
 537             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 538             return;
 539         }
 540         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 541             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 542             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 543             final ReentrantLock lock = lockFor(lriKey);
 544             lock.lock();
 545             try {
 546                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 547                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 548
 549                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 550                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 551                                 fibUtil.getVpnInstanceOpData(rd);
 552                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 553                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 554                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 555                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 556                             }
 557                         }
 558                     }
 559                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 560                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 561                 }
 562             } finally {
 563                 lock.unlock();
 564             }
 565         });
 566         final List<InstructionInfo> instructions = new ArrayList<>();
 567         Uint64 subnetRouteMeta = Uint64.valueOf(BigInteger.valueOf(elanTag).shiftLeft(24)
 568             .or(BigInteger.valueOf(vpnId.longValue()).shiftLeft(1)));
 569         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 570         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 571         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 572                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 573         if (vrfEntry.getRoutePaths() != null) {
 574             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 575                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 576                     List<ActionInfo> actionsInfos = new ArrayList<>();
 577                     // reinitialize instructions list for LFIB Table
 578                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 579                     actionsInfos.add(new ActionPopMpls(etherType));
 580                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 581                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 582                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 583                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 584
 585                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions,
 586                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 587                 }
 588             }
 589         }
 590     }
 591
 592     private void installSubnetBroadcastAddrDropRule(final Uint64 dpnId, final String rd, final Uint32 vpnId,
 593             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 594         List<MatchInfo> matches = new ArrayList<>();
 595
 596         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 597                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 598         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 599         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 600         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 601
 602         InetAddress destPrefix;
 603         try {
 604             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 605         } catch (UnknownHostException e) {
 606             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 607                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 608             return;
 609         }
 610
 611         // Match on VpnId and SubnetBroadCast IP address
 612         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 613             MetaDataUtil.METADATA_MASK_VRFID));
 614         matches.add(MatchEthernetType.IPV4);
 615
 616         if (prefixLength != 0) {
 617             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 618         }
 619
 620         //Action is to drop the packet
 621         List<InstructionInfo> dropInstructions = new ArrayList<>();
 622         List<ActionInfo> actionsInfos = new ArrayList<>();
 623         actionsInfos.add(new ActionDrop());
 624         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 625
 626         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 627         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 628         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 629                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 630
 631         Flow flow = flowEntity.getFlowBuilder().build();
 632         String flowId = flowEntity.getFlowId();
 633         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 634         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 635
 636         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 637                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 638                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 639
 640         if (addOrRemove == NwConstants.ADD_FLOW) {
 641             tx.put(flowInstanceId,flow, true);
 642         } else {
 643             tx.delete(flowInstanceId);
 644         }
 645     }
 646
 647     /*
 648      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 649      * LportDispatcher table (via table 80)
 650      */
 651     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 652                                             final VrfEntry vrfEntry, int etherType) {
 653         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 654         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 655         // packet is commuted from Vpn2 to Vpn1.
 656         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 657         if (!interVpnLink.isActive()) {
 658             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 659                      interVpnLinkName, vrfEntry.getDestPrefix());
 660             return;
 661         }
 662
 663         Optional<Uint32> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 664         if (!optLportTag.isPresent()) {
 665             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 666             return;
 667         }
 668
 669         Long lportTag = optLportTag.get().toJava();
 670         Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 671         if (label == null) {
 672             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 673                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 674             return;
 675         }
 676         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 677         List<InstructionInfo> instructions = Arrays.asList(
 678             new InstructionApplyActions(actionsInfos),
 679             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 680                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 681                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 682                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 683             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 684         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 685         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 686
 687         for (Uint64 dpId : targetDpns) {
 688             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 689                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 690
 691             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 692                                /*writeTx*/null);
 693         }
 694     }
 695
 696
 697     /*
 698      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 699      */
 700     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 701                                                final VrfEntry vrfEntry, Uint32 vpnTag) {
 702         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 703         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 704             && vrfEntry.getRoutePaths().size() == 1);
 705         String destination = vrfEntry.getDestPrefix();
 706         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 707         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 708
 709         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 710         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 711         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 712         if (interVpnLink.getState().or(State.Error) != State.Active) {
 713             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 714                 destination, nextHop, interVpnLinkName);
 715             return;
 716         }
 717
 718         Optional<Uint32> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 719         if (!optOtherEndpointLportTag.isPresent()) {
 720             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 721                 vpnUuid, interVpnLinkName);
 722             return;
 723         }
 724
 725         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 726         if (targetDpns.isEmpty()) {
 727             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 728                 vpnUuid, interVpnLinkName);
 729             return;
 730         }
 731
 732         String[] values = destination.split("/");
 733         String destPrefixIpAddress = values[0];
 734         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 735
 736         List<MatchInfo> matches = new ArrayList<>();
 737         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag.longValue()),
 738             MetaDataUtil.METADATA_MASK_VRFID));
 739         matches.add(MatchEthernetType.IPV4);
 740
 741         if (prefixLength != 0) {
 742             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 743         }
 744
 745         List<Instruction> instructions =
 746             Arrays.asList(new InstructionWriteMetadata(
 747                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 748                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 749                             .L3VPN_SERVICE_INDEX)),
 750                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 751                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 752
 753         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 754         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 755         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 756             COOKIE_VM_FIB_TABLE, matches, instructions);
 757
 758         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 759             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 760
 761         for (Uint64 dpId : targetDpns) {
 762
 763             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 764                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 765                 dpId, interVpnLink.getInterVpnLinkName());
 766
 767             mdsalManager.installFlow(dpId, flowEntity);
 768         }
 769     }
 770
 771     private List<Uint64> createLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 772         List<Uint64> returnLocalDpnId = new ArrayList<>();
 773         String localNextHopIP = vrfEntry.getDestPrefix();
 774         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 775         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 776         if (localNextHopInfo == null) {
 777             boolean localNextHopSeen = false;
 778             List<Routes> vpnExtraRoutes = null;
 779             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 780             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 781             // FIXME: use an Identifier structure?
 782             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 783             lock.lock();
 784             try {
 785                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 786                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 787                         vpnName, usedRds, localNextHopIP);
 788                 if (LOG.isDebugEnabled()) {
 789                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 790                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 791                 }
 792
 793                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 794                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 795                     String ipPrefix;
 796                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 797                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 798                     } else {
 799                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 800                     }
 801                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 802                             ipPrefix);
 803                     if (localNextHopInfoLocal != null) {
 804                         localNextHopSeen = true;
 805                         Uint64 dpnId =
 806                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 807                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType,
 808                                         /*parentVpnId*/ null);
 809                         returnLocalDpnId.add(dpnId);
 810                     }
 811                 }
 812             } finally {
 813                 lock.unlock();
 814             }
 815             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 816                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 817                 if (optionalLabel.isPresent()) {
 818                     Uint32 label = optionalLabel.get();
 819                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 820                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 821                     final ReentrantLock labelLock = lockFor(lriKey);
 822                     labelLock.lock();
 823                     try {
 824                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 825                         Uint32 parentVpnId = lri.getParentVpnid();
 826                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 827                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 828                                     fibUtil.getVpnInstanceOpData(rd);
 829                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 830                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 831                                 if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains(
 832                                        vpnInstanceName)) {
 833                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 834                                     localNextHopIP = lri.getPrefix();
 835                                 } else {
 836                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 837                                     localNextHopIP = lri.getPrefix();
 838                                 }
 839                             }
 840                             if (localNextHopInfo != null) {
 841                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 842                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 843                                 if (vpnExtraRoutes.isEmpty()) {
 844                                     Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 845                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType, parentVpnId);
 846                                     returnLocalDpnId.add(dpnId);
 847                                 } else {
 848                                     for (Routes extraRoutes : vpnExtraRoutes) {
 849                                         Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 850                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType,
 851                                                 parentVpnId);
 852                                         returnLocalDpnId.add(dpnId);
 853                                     }
 854                                 }
 855                             }
 856                         }
 857                     } finally {
 858                         labelLock.unlock();
 859                     }
 860                 }
 861             }
 862             if (returnLocalDpnId.isEmpty()) {
 863                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 864             }
 865         } else {
 866             Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 867                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType,
 868                     /*parentVpnId*/ null);
 869             if (dpnId != null) {
 870                 returnLocalDpnId.add(dpnId);
 871             }
 872         }
 873         return returnLocalDpnId;
 874     }
 875
 876     private Uint64 checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 877                                                 final Uint32 vpnId, final String rd,
 878                                                 final VrfEntry vrfEntry,
 879                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 880                                                 int etherType, Uint32 parentVpnId) {
 881         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 882         if (localNextHopInfo != null) {
 883             long groupId;
 884             long localGroupId;
 885             final Uint64 dpnId = localNextHopInfo.getDpnId();
 886             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 887                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 888                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 889                 return dpnId;
 890             }
 891             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 892                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 893                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 894                 return dpnId;
 895             }
 896             if (!isVpnPresentInDpn(rd, dpnId)) {
 897                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 898                         vpnId, rd, dpnId.toString());
 899                 return Uint64.ZERO;
 900             }
 901             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 902             String prefix = vrfEntry.getDestPrefix();
 903             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 904             //The loadbalancing group is created only if the extra route has multiple nexthops
 905             //to avoid loadbalancing the discovered routes
 906             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 907                     && routes != null) {
 908                 if (vpnExtraRoutes.size() > 1) {
 909                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 910                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 911                 } else {
 912                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 913                     localGroupId = groupId;
 914                 }
 915             } else {
 916                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 917                         gwMacAddress, parentVpnId);
 918                 localGroupId = groupId;
 919             }
 920             if (groupId == FibConstants.INVALID_GROUP_ID) {
 921                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 922                         prefix, rd, interfaceName, dpnId.toString());
 923                 return Uint64.ZERO;
 924             }
 925             final List<InstructionInfo> instructions = Collections.singletonList(
 926                     new InstructionApplyActions(
 927                             Collections.singletonList(new ActionGroup(groupId))));
 928             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 929                     new InstructionApplyActions(
 930                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 931             java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 932             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 933             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 934             jobCoordinator.enqueueJob(jobKey,
 935                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 936                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 937                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 938                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 939                         optLabel.ifPresent(label -> {
 940                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 941                                 LOG.debug(
 942                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 943                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 944                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 945                                         nextHopAddressList);
 946                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 947                                         NwConstants.ADD_FLOW, tx);
 948                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 949                             } else {
 950                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 951                                                 + "route. LFib and Terminating table entries will not be created.",
 952                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 953                             }
 954                         });
 955                     }
 956                 })));
 957             return dpnId;
 958         }
 959         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 960                 vpnName);
 961         return Uint64.ZERO;
 962     }
 963
 964     private boolean isVpnPresentInDpn(String rd, Uint64 dpnId)  {
 965         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 966         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 967         return dpnInVpn.isPresent();
 968     }
 969
 970     @Nullable
 971     private LabelRouteInfo getLabelRouteInfo(Uint32 label) {
 972         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 973     }
 974
 975     @Nullable
 976     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 977         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 978             .child(LabelRouteInfo.class, label).build();
 979         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 980         if (opResult.isPresent()) {
 981             return opResult.get();
 982         }
 983         return null;
 984     }
 985
 986     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 987             @Nullable TypedWriteTransaction<Operational> tx) {
 988         if (lri == null) {
 989             return true;
 990         }
 991
 992         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 993         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 994             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 995
 996         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 997             ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 998         if (vpnInstancesList.contains(vpnInstanceName)) {
 999             LOG.debug("vpninstance {} name is present", vpnInstanceName);
1000             vpnInstancesList.remove(vpnInstanceName);
1001         }
1002         if (vpnInstancesList.isEmpty()) {
1003             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
1004             if (tx != null) {
1005                 tx.delete(lriId);
1006             } else {
1007                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
1008             }
1009             return true;
1010         } else {
1011             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
1012             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
1013             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1014         }
1015         return false;
1016     }
1017
1018     void makeTunnelTableEntry(Uint64 dpId, Uint32 label, long groupId/*String egressInterfaceName*/,
1019                                       TypedWriteTransaction<Configuration> tx) {
1020         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1021
1022         createTerminatingServiceActions(dpId, label, actionsInfos, tx);
1023
1024         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1025             dpId, label, groupId);
1026     }
1027
1028     public void createTerminatingServiceActions(Uint64 destDpId, Uint32 label, List<ActionInfo> actionsInfos,
1029                                                 TypedWriteTransaction<Configuration> tx) {
1030         List<MatchInfo> mkMatches = new ArrayList<>();
1031
1032         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1033             destDpId, label, actionsInfos);
1034
1035         // Matching metadata
1036         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1037         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1038
1039         List<InstructionInfo> mkInstructions = new ArrayList<>();
1040         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1041
1042         FlowEntity terminatingServiceTableFlowEntity =
1043             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1044             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1045                     FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1046                     String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1047                     Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()),
1048                     mkMatches, mkInstructions);
1049
1050         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1051
1052         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1053
1054         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1055         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1056             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1057             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1058             .child(Flow.class, flowKey).build();
1059         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1060     }
1061
1062     private void removeTunnelTableEntry(Uint64 dpId, Uint32 label, TypedWriteTransaction<Configuration> tx) {
1063         FlowEntity flowEntity;
1064         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1065         List<MatchInfo> mkMatches = new ArrayList<>();
1066         // Matching metadata
1067         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1068         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1069             NwConstants.INTERNAL_TUNNEL_TABLE,
1070             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1071                 FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1072                 String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1073                 Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()), mkMatches, null);
1074         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1075         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1076         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1077             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1078             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1079
1080         tx.delete(flowInstanceId);
1081         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1082     }
1083
1084     public List<Uint64> deleteLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry) {
1085         List<Uint64> returnLocalDpnId = new ArrayList<>();
1086         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1087         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1088         boolean shouldUpdateNonEcmpLocalNextHop = true;
1089         if (localNextHopInfo == null) {
1090             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1091             if (usedRds.size() > 1) {
1092                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1093                         vrfEntry.getDestPrefix(), vpnName, rd);
1094                 return returnLocalDpnId;
1095             }
1096             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1097             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1098             //in the vpn
1099             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1100                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1101             if (extraRouteOptional.isPresent()) {
1102                 Routes extraRoute = extraRouteOptional.get();
1103                 String ipPrefix;
1104                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1105                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1106                 } else {
1107                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1108                 }
1109                 if (extraRoute.getNexthopIpList().size() > 1) {
1110                     shouldUpdateNonEcmpLocalNextHop = false;
1111                 }
1112                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1113                 if (localNextHopInfo != null) {
1114                     String localNextHopIP = localNextHopInfo.getIpAddress();
1115                     Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1116                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1117                     if (!dpnId.equals(Uint64.ZERO)) {
1118                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1119                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1120                         returnLocalDpnId.add(dpnId);
1121                     }
1122                 } else {
1123                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1124                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1125                 }
1126             }
1127
1128             if (localNextHopInfo == null) {
1129                 /* Imported VRF entry */
1130                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1131                 if (optionalLabel.isPresent()) {
1132                     Uint32 label = optionalLabel.get();
1133                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1134                     LabelRouteInfo lri = getLabelRouteInfo(label);
1135                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1136                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1137                         prefixBuilder.setDpnId(lri.getDpnId());
1138                         Uint64 dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1139                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1140                         if (!dpnId.equals(Uint64.ZERO)) {
1141                             returnLocalDpnId.add(dpnId);
1142                         }
1143                     }
1144                 }
1145             }
1146
1147         } else {
1148             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1149             String localNextHopIP = localNextHopInfo.getIpAddress();
1150             Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1151                     shouldUpdateNonEcmpLocalNextHop);
1152             if (!dpnId.equals(Uint64.ZERO)) {
1153                 returnLocalDpnId.add(dpnId);
1154             }
1155         }
1156
1157         return returnLocalDpnId;
1158     }
1159
1160     private Uint64 checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1161             final String vpnName, final Uint32 vpnId, final String rd, final VrfEntry vrfEntry,
1162             boolean shouldUpdateNonEcmpLocalNextHop) {
1163         if (localNextHopInfo != null) {
1164             final Uint64 dpnId = localNextHopInfo.getDpnId();
1165             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1166                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1167                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1168                 return dpnId;
1169             }
1170             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1171                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1172                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1173                 return dpnId;
1174             }
1175
1176             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1177                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1178                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1179                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1180                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1181                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1182                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1183                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1184                                         NwConstants.DEL_FLOW, tx);
1185                                 removeTunnelTableEntry(dpnId, label, tx);
1186                             });
1187                         }
1188                     }
1189                 })));
1190             //TODO: verify below adjacency call need to be optimized (?)
1191             //In case of the removal of the extra route, the loadbalancing group is updated
1192             if (shouldUpdateNonEcmpLocalNextHop) {
1193                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1194             }
1195             return dpnId;
1196         }
1197         return Uint64.ZERO;
1198     }
1199
1200     private void createRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1201             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1202         if (tx == null) {
1203             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1204                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1205                 "Error creating remote FIB entry");
1206             return;
1207         }
1208
1209         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1210         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", vrfEntry.getDestPrefix(), rd,
1211                 remoteDpnId);
1212
1213         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1214             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1215             return;
1216         }
1217         // Handling static VRF entries
1218         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1219         List<Routes> vpnExtraRoutes =
1220                 VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, vpnName, usedRds, vrfEntry.getDestPrefix());
1221         if (!vpnExtraRoutes.isEmpty()) {
1222             programRemoteFibWithLoadBalancingGroups(remoteDpnId, vpnId, rd, vrfEntry, vpnExtraRoutes);
1223         } else {
1224             // Program in case of other static VRF entries like floating IPs
1225             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1226         }
1227     }
1228
1229     // Allow deprecated TransactionRunner calls for now
1230     @SuppressWarnings("ForbidCertainMethod")
1231     private void programRemoteFibWithLoadBalancingGroups(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1232             final VrfEntry vrfEntry, List<Routes> vpnExtraRoutes) {
1233         // create loadbalancing groups for extra routes only when the extra route is
1234         // present behind multiple VMs
1235         // Obtain the local routes for this particular dpn.
1236         java.util.Optional<Routes> routes = vpnExtraRoutes.stream().filter(route -> {
1237             Prefixes prefixToInterface =
1238                     fibUtil.getPrefixToInterface(vpnId, FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1239             if (prefixToInterface == null) {
1240                 return false;
1241             }
1242             return remoteDpnId.equals(prefixToInterface.getDpnId());
1243         }).findFirst();
1244         long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1245                 routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1246         if (groupId == FibConstants.INVALID_GROUP_ID) {
1247             LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", vrfEntry.getDestPrefix(), rd,
1248                     remoteDpnId);
1249             return;
1250         }
1251         List<ActionInfo> actionInfos = Collections.singletonList(new ActionGroup(groupId));
1252         List<InstructionInfo> instructions = Lists.newArrayList(new InstructionApplyActions(actionInfos));
1253         String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1254         jobCoordinator.enqueueJob(jobKey,
1255             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1256                 baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1257                         NwConstants.ADD_FLOW, txn, null);
1258             })));
1259
1260         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1261     }
1262
1263     private void programRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1264             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1265         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1266         if (adjacencyResults.isEmpty()) {
1267             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", vrfEntry.getRoutePaths(), rd,
1268                     remoteDpnId);
1269             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1270             return;
1271         }
1272         baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, TransactionAdapter.toWriteTransaction(tx),
1273                 rd, adjacencyResults, null);
1274         LOG.debug("Successfully programmed FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1275     }
1276
1277     protected void cleanUpOpDataForFib(Uint32 vpnId, String primaryRd, final VrfEntry vrfEntry) {
1278     /* Get interface info from prefix to interface mapping;
1279         Use the interface info to get the corresponding vpn interface op DS entry,
1280         remove the adjacency corresponding to this fib entry.
1281         If adjacency removed is the last adjacency, clean up the following:
1282          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1283          - prefix to interface entry
1284          - vpn interface op DS
1285      */
1286         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1287         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1288         if (prefixInfo == null) {
1289             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1290             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1291             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1292             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
1293                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1294                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1295                     if (nextHopIp != null) {
1296                         String ipPrefix;
1297                         if (isIpv4Address(nextHopIp)) {
1298                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1299                         } else {
1300                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1301                         }
1302                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1303                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1304                     }
1305                 }
1306             }
1307             if (prefixInfo == null) {
1308                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1309                 if (optionalLabel.isPresent()) {
1310                     Uint32 label = optionalLabel.get();
1311                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1312                     LabelRouteInfo lri = getLabelRouteInfo(label);
1313                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1314                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1315                         prefixBuilder.setDpnId(lri.getDpnId());
1316                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1317                         prefixBuilder.setIpAddress(lri.getPrefix());
1318                         prefixInfo = prefixBuilder.build();
1319                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1320                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1321                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1322                     }
1323                 }
1324             }
1325         } else {
1326             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1327         }
1328     }
1329
1330     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1331                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1332
1333         if (prefixInfo == null) {
1334             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1335                     vrfEntry.getDestPrefix(), vpnId, rd);
1336             return; //Don't have any info for this prefix (shouldn't happen); need to return
1337         }
1338
1339         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1340             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1341                     vrfEntry.getDestPrefix(), vpnId, rd);
1342             return;
1343         }
1344
1345         String ifName = prefixInfo.getVpnInterfaceName();
1346         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1347             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1348     }
1349
1350     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1351         Prefixes prefixInfo;
1352         Uint32 vpnId;
1353         String rd;
1354         VrfEntry vrfEntry;
1355         Routes extraRoute;
1356
1357         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1358                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1359             this.prefixInfo = prefixInfo;
1360             this.vpnId = vpnId;
1361             this.rd = rd;
1362             this.vrfEntry = vrfEntry;
1363             this.extraRoute = extraRoute;
1364         }
1365
1366         @Override
1367         public List<ListenableFuture<Void>> call() {
1368             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1369             // to call the respective helpers.
1370             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1371                 //First Cleanup LabelRouteInfo
1372                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1373                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1374                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1375                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1376                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1377                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1378                         final ReentrantLock lock = lockFor(lriKey);
1379                         lock.lock();
1380                         try {
1381                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1382                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1383                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1384                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1385                                         fibUtil.getVpnInstanceOpData(rd);
1386                                 String vpnInstanceName = "";
1387                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1388                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1389                                 }
1390                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1391                                 if (lriRemoved) {
1392                                     String parentRd = lri.getParentVpnRd();
1393                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1394                                             parentRd, vrfEntry.getDestPrefix()));
1395                                 }
1396                             } else {
1397                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1398                                         rd, vrfEntry.getDestPrefix()));
1399                             }
1400                         } finally {
1401                             lock.unlock();
1402                         }
1403                     });
1404                 }
1405                 String ifName = prefixInfo.getVpnInterfaceName();
1406                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1407                 String vpnName = null;
1408
1409                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1410                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1411                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1412                     if (vpnNameOpt.isPresent()) {
1413                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1414                     }
1415                 }
1416                 if (optVpnName.isPresent()) {
1417                     vpnName = optVpnName.get();
1418                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1419                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1420                     if (opVpnInterface.isPresent()) {
1421                         Uint32 associatedVpnId = fibUtil.getVpnId(vpnName);
1422                         if (!Objects.equals(vpnId, associatedVpnId)) {
1423                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1424                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1425                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1426                             return;
1427                         } else {
1428                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1429                                     vrfEntry.getDestPrefix(), associatedVpnId);
1430                         }
1431                     }
1432                 }
1433                 if (extraRoute != null) {
1434                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1435                     //Only one used Rd present in case of removal event
1436                     String usedRd = usedRds.get(0);
1437                     if (optVpnName.isPresent()) {
1438                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1439                                         vrfEntry.getDestPrefix()));
1440                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1441                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1442                     }
1443                 }
1444                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1445             }));
1446         }
1447     }
1448
1449     /**
1450      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1451      * Remove Adjacency from VPNInterfaceOpData.
1452      * if Adjacency != primary.
1453      * if Adjacency == primary , then mark it for deletion.
1454      * Remove entire VPNinterfaceOpData Entry.
1455      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1456      * @param vrfEntry - VrfEntry removed
1457      * @param ifName - Interface name from VRFentry
1458      * @param vpnName - VPN name of corresponding VRF
1459      * @param tx - ReadWrite Tx
1460      */
1461     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
1462             justification = "https://github.com/spotbugs/spotbugs/issues/811")
1463     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1464                                                           TypedReadWriteTransaction<Operational> tx)
1465             throws ExecutionException, InterruptedException {
1466         InstanceIdentifier<Adjacency> adjacencyIid =
1467                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1468         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1469         if (adjacencyOptional.isPresent()) {
1470             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1471                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1472             } else {
1473                 tx.merge(adjacencyIid,
1474                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1475             }
1476         }
1477
1478         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1479
1480         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1481             return;
1482         }
1483
1484         @NonNull List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
1485         if (adjacencies.size() <= 2
1486                 && adjacencies.stream().allMatch(adjacency ->
1487                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1488                         && adjacency.isMarkedForDeletion() != null
1489                         && adjacency.isMarkedForDeletion()
1490         )) {
1491             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1492             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1493         }
1494     }
1495
1496     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1497         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1498         final String rd = vrfTableKey.getRouteDistinguisher();
1499         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1500         if (vpnInstance == null) {
1501             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1502             return;
1503         }
1504         final Collection<VpnToDpnList> vpnToDpnList;
1505         if (vrfEntry.getParentVpnRd() != null
1506                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1507             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1508             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1509             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1510                     vpnInstance.getVpnToDpnList();
1511             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1512                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1513         } else {
1514             vpnToDpnList = vpnInstance.getVpnToDpnList();
1515         }
1516
1517         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1518         final java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1519         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1520         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1521         if (subnetRoute != null) {
1522             long elanTag = subnetRoute.getElantag().toJava();
1523             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1524                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1525             if (vpnToDpnList != null) {
1526                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1527                     () -> Collections.singletonList(
1528                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1529                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1530
1531                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(),
1532                                     vpnInstance.getVpnId(),
1533                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1534                                     TransactionAdapter.toWriteTransaction(tx), null);
1535                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1536                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(),
1537                                         label, null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1538                                 }
1539
1540                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
1541                                     vpnInstance.getVpnId(),
1542                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1543                             }
1544                         })));
1545             }
1546             optionalLabel.ifPresent(label -> {
1547                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1548                 final ReentrantLock lock = lockFor(lriKey);
1549                 lock.lock();
1550                 try {
1551                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1552                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1553                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1554                                 fibUtil.getVpnInstanceOpData(rd);
1555                         String vpnInstanceName = "";
1556                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1557                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1558                         }
1559                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1560                         if (lriRemoved) {
1561                             String parentRd = lri.getParentVpnRd();
1562                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1563                                     parentRd, vrfEntry.getDestPrefix()));
1564                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1565                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1566                         }
1567                     } else {
1568                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1569                                 rd, vrfEntry.getDestPrefix()));
1570                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1571                                 label, rd, vrfEntry.getDestPrefix());
1572                     }
1573                 } finally {
1574                     lock.unlock();
1575                 }
1576             });
1577             return;
1578         }
1579
1580         final List<Uint64> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1581             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1582         if (vpnToDpnList != null) {
1583             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1584                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1585             String jobKey;
1586             Optional<Routes> extraRouteOptional;
1587             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1588             if (usedRds != null && !usedRds.isEmpty()) {
1589                 if (usedRds.size() > 1) {
1590                     LOG.error("The extra route prefix is still present in some DPNs");
1591                     return ;
1592                 } else {
1593                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1594                     //is not present in any other DPN
1595                     extraRouteOptional = VpnExtraRouteHelper
1596                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1597                 }
1598             } else {
1599                 extraRouteOptional = Optional.absent();
1600             }
1601
1602             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1603                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1604                     if (localDpnIdList.size() <= 0) {
1605                         for (VpnToDpnList curDpn : vpnToDpnList) {
1606                             baseVrfEntryHandler.deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(),
1607                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1608                                 TransactionAdapter.toWriteTransaction(tx));
1609                         }
1610                     } else {
1611                         for (Uint64 localDpnId : localDpnIdList) {
1612                             for (VpnToDpnList curDpn : vpnToDpnList) {
1613                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1614                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1615                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1616                                         TransactionAdapter.toWriteTransaction(tx));
1617                                 }
1618                             }
1619                         }
1620                     }
1621                     if (extraRouteOptional.isPresent()) {
1622                         //Remove select groups only for extra-routes
1623                         nextHopManager.removeNextHopPointer(nextHopManager
1624                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1625                         nextHopManager.removeNextHopPointer(nextHopManager
1626                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1627                     }
1628                 })), MAX_RETRIES);
1629         }
1630
1631         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1632         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1633         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1634
1635         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1636         // of the interVpnLink.
1637         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1638         if (optVpnUuid.isPresent()) {
1639             String vpnUuid = optVpnUuid.get();
1640             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1641                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1642                 if (optInterVpnLink.isPresent()) {
1643                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1644                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1645                         // This is route that points to the other endpoint of an InterVpnLink
1646                         // In that case, we should look for the FIB table pointing to
1647                         // LPortDispatcher table and remove it.
1648                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1649                     }
1650                 }
1651             });
1652         }
1653
1654     }
1655
1656     private void makeLFibTableEntry(Uint64 dpId, Uint32 label, @Nullable List<InstructionInfo> instructions,
1657                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1658         if (tx == null) {
1659             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1660                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1661                 "Error making LFIB table entry");
1662             return;
1663         }
1664
1665         List<MatchInfo> matches = new ArrayList<>();
1666         matches.add(MatchEthernetType.MPLS_UNICAST);
1667         matches.add(new MatchMplsLabel(label.longValue()));
1668
1669         // Install the flow entry in L3_LFIB_TABLE
1670         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1671
1672         FlowEntity flowEntity;
1673         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1674             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1675         Flow flow = flowEntity.getFlowBuilder().build();
1676         String flowId = flowEntity.getFlowId();
1677         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1678         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1679         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1680             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1681             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1682
1683         if (addOrRemove == NwConstants.ADD_FLOW) {
1684             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1685         } else {
1686             tx.delete(flowInstanceId);
1687         }
1688
1689         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1690             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1691     }
1692
1693     public void populateFibOnNewDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1694                                     final FutureCallback<List<Void>> callback) {
1695         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1696         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1697             () -> {
1698                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1699                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1700                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1701                         LogicalDatastoreType.CONFIGURATION, id);
1702                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1703                 if (!vrfTable.isPresent()) {
1704                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1705                     if (callback != null) {
1706                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1707                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1708                     }
1709                     return futures;
1710                 }
1711
1712                 final ReentrantLock lock = lockFor(vpnInstance);
1713                 lock.lock();
1714                 try {
1715                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1716                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1717                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1718                             if (subnetRoute != null) {
1719                                 long elanTag = subnetRoute.getElantag().toJava();
1720                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1721                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1722                                         tx);
1723                                 continue;
1724                             }
1725                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1726                             if (routerInt != null) {
1727                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1728                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1729                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1730                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1731                                         NwConstants.ADD_FLOW);
1732                                 continue;
1733                             }
1734                             //Handle local flow creation for imports
1735                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1736                                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1737                                 if (optionalLabel.isPresent()) {
1738                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1739                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1740                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1741                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1742                                             try {
1743                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1744                                                         vrfEntry.getDestPrefix());
1745                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1746                                             } catch (IllegalArgumentException ex) {
1747                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1748                                                         vrfEntry.getDestPrefix());
1749                                             }
1750                                             continue;
1751                                         }
1752                                     }
1753                                 }
1754                             }
1755                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1756                                     vrfEntry, dpnId);
1757                             if (shouldCreateRemoteFibEntry) {
1758                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1759                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1760                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1761                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1762                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1763                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1764                                 } else {
1765                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1766                                             vrfEntry, tx);
1767                                 }
1768                             }
1769                         }
1770                     }));
1771                     if (callback != null) {
1772                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1773                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1774                     }
1775                 } finally {
1776                     lock.unlock();
1777                 }
1778                 return futures;
1779             });
1780     }
1781
1782     public void populateExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1783                                             final String localNextHopIp, final String remoteNextHopIp) {
1784         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1785             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1786         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1787         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1788         List<SubTransaction> txnObjects =  new ArrayList<>();
1789         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1790         if (vrfTable.isPresent()) {
1791             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1792                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1793                     final ReentrantLock lock = lockFor(vpnInstance);
1794                     lock.lock();
1795                     try {
1796                         vrfTable.get().nonnullVrfEntry().stream()
1797                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1798                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1799                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1800                     } finally {
1801                         lock.unlock();
1802                     }
1803                 })));
1804         }
1805     }
1806
1807     public void manageRemoteRouteOnDPN(final boolean action,
1808                                        final Uint64 localDpnId,
1809                                        final Uint32 vpnId,
1810                                        final String rd,
1811                                        final String destPrefix,
1812                                        final String destTepIp,
1813                                        final Uint32 label) {
1814         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1815
1816         if (vpnInstance == null) {
1817             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1818             return;
1819         }
1820
1821         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1822             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1823                 final ReentrantLock lock = lockFor(vpnInstance);
1824                 lock.lock();
1825                 try {
1826                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1827                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1828                     if (vrfEntry == null) {
1829                         return;
1830                     }
1831                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1832                             action, localDpnId, vpnId, rd, destPrefix);
1833                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1834                     VrfEntry modVrfEntry;
1835                     if (routePathList == null || routePathList.isEmpty()) {
1836                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1837                                 Collections.singletonList(destTepIp),
1838                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1839                     } else {
1840                         modVrfEntry = vrfEntry;
1841                     }
1842
1843                     if (action) {
1844                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1845                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1846                                 modVrfEntry, tx);
1847                     } else {
1848                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1849                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1850                                 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1851                         if (usedRds.size() > 1) {
1852                             LOG.debug("The extra route prefix is still present in some DPNs");
1853                             return;
1854                         }
1855                         //Is this fib route an extra route? If yes, get the nexthop which would be
1856                         //an adjacency in the vpn
1857                         Optional<Routes> extraRouteOptional = Optional.absent();
1858                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC && usedRds.size() != 0) {
1859                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1860                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1861                                     usedRds.get(0), vrfEntry.getDestPrefix());
1862                         }
1863                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1864                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1865                     }
1866                 } finally {
1867                     lock.unlock();
1868                 }
1869             })));
1870     }
1871
1872     public void cleanUpDpnForVpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1873                                  final FutureCallback<List<Void>> callback) {
1874         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1875         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1876             () -> {
1877                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1878                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1879                 List<SubTransaction> txnObjects = new ArrayList<>();
1880                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1881                         LogicalDatastoreType.CONFIGURATION, id);
1882                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1883                 if (!vrfTable.isPresent()) {
1884                     LOG.error("cleanUpDpnForVpn: VRF Table not available for RD {}", rd);
1885                     if (callback != null) {
1886                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1887                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1888                     }
1889                     return futures;
1890                 }
1891                 final ReentrantLock lock = lockFor(vpnInstance);
1892                 lock.lock();
1893                 try {
1894                     futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1895                         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1896                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry()) {
1897                             /* parentRd is only filled for external PNF cases where the interface on the external
1898                              * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1899                              * #fibUtil.isInterfacePresentInDpn().
1900                              * */
1901                             String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1902                                     : rd;
1903                             /* Handle subnet routes here */
1904                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1905                             if (subnetRoute != null && !fibUtil
1906                                     .isInterfacePresentInDpn(parentRd, dpnId)) {
1907                                 LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1908                                         + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1909                                 baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1910                                         NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1911                                 List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1912                                 if (routePaths != null) {
1913                                     for (RoutePaths routePath : routePaths) {
1914                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1915                                                 DEFAULT_FIB_FLOW_PRIORITY,
1916                                                 NwConstants.DEL_FLOW, tx);
1917                                         LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1918                                                         + " for rd {} prefix {}", routePath.getLabel(), rd,
1919                                                 vrfEntry.getDestPrefix());
1920                                     }
1921                                 }
1922                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1923                                         NwConstants.DEL_FLOW, tx);
1924                                 continue;
1925                             }
1926                             // ping responder for router interfaces
1927                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1928                             if (routerInt != null) {
1929                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1930                                         rd, routerInt.getUuid(), routerInt.getIpAddress(),
1931                                         routerInt.getMacAddress());
1932                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1933                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1934                                         NwConstants.DEL_FLOW);
1935                                 continue;
1936                             }
1937                             //Handle local flow deletion for imports
1938                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1939                                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1940                                 if (optionalLabel.isPresent()) {
1941                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1942                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1943                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1944                                             lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1945                                         deleteLocalFibEntry(vpnId, rd, vrfEntry);
1946                                     }
1947                                 }
1948                             }
1949                             // Passing null as we don't know the dpn
1950                             // to which prefix is attached at this point
1951                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1952                                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1953                             Optional<Routes> extraRouteOptional;
1954                             //Is this fib route an extra route? If yes, get the nexthop which would be
1955                             //an adjacency in the vpn
1956                             if (usedRds != null && !usedRds.isEmpty()) {
1957                                 if (usedRds.size() > 1) {
1958                                     LOG.error("The extra route prefix is still present in some DPNs");
1959                                     return;
1960                                 } else {
1961                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1962                                             usedRds.get(0), vrfEntry.getDestPrefix());
1963
1964                                 }
1965                             } else {
1966                                 extraRouteOptional = Optional.absent();
1967                             }
1968                             if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1969                                 bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1970                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1971                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1972                             } else {
1973                                 if (subnetRoute == null || !fibUtil
1974                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
1975                                     baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1976                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1977                                             TransactionAdapter.toWriteTransaction(tx));
1978                                 }
1979                             }
1980                         }
1981                     }));
1982                 } finally {
1983                     lock.unlock();
1984                 }
1985                 if (callback != null) {
1986                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1987                     Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1988                 }
1989                 return futures;
1990             });
1991     }
1992
1993     public void cleanUpExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1994                                            final String localNextHopIp, final String remoteNextHopIp) {
1995         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1996                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1997             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1998         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1999         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
2000         List<SubTransaction> txnObjects =  new ArrayList<>();
2001         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2002         if (vrfTable.isPresent()) {
2003             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
2004                 () -> {
2005                     final ReentrantLock lock = lockFor(vpnInstance);
2006                     lock.lock();
2007                     try {
2008                         return Collections.singletonList(
2009                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
2010                                 tx -> vrfTable.get().nonnullVrfEntry().stream()
2011                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2012                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
2013                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
2014                                         txnObjects))));
2015                     } finally {
2016                         lock.unlock();
2017                     }
2018                 });
2019         }
2020     }
2021
2022     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2023         InstanceIdentifierBuilder<VrfTables> idBuilder =
2024             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2025         return idBuilder.build();
2026     }
2027
2028     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2029         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2030                 .FLOWID_SEPARATOR + nextHop;
2031     }
2032
2033     private String getTableMissFlowRef(Uint64 dpnId, short tableId, Uint32 tableMiss) {
2034         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2035                 + tableMiss + FLOWID_PREFIX;
2036     }
2037
2038     @Nullable
2039     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2040         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2041             .child(VrfTables.class, new VrfTablesKey(rd))
2042             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2043         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2044         if (vrfEntry.isPresent()) {
2045             return vrfEntry.get();
2046         }
2047         return null;
2048     }
2049
2050     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2051                                              final String vpnName,
2052                                              final VrfEntry vrfEntry) {
2053         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2054
2055         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2056         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2057
2058         if (targetDpns.isEmpty()) {
2059             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2060             return;
2061         }
2062
2063         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2064         java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2065
2066         // delete from FIB
2067         //
2068         optNextHop.ifPresent(nextHop -> {
2069             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2070             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2071             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2072                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2073
2074             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2075             for (Uint64 dpId : targetDpns) {
2076                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2077                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2078
2079                 mdsalManager.removeFlow(dpId, flow);
2080             }
2081         });
2082
2083         // delete from LFIB
2084         //
2085         optLabel.ifPresent(label -> {
2086             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2087
2088             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2089                 for (Uint64 dpId : targetDpns) {
2090                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2091                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2092                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY,
2093                             NwConstants.DEL_FLOW, tx);
2094                 }
2095             }), LOG, "Error removing flows");
2096         });
2097     }
2098
2099     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2100             List<String> nextHopAddressList, LabelRouteInfo lri) {
2101         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2102                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2103     }
2104
2105     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Uint32 vpnId, VrfEntry vrfEntry, Uint64 dpnId) {
2106         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2107             return true;
2108         }
2109
2110         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2111         if (prefix != null) {
2112             Uint64 prefixDpnId = prefix.getDpnId();
2113             if (dpnId.equals(prefixDpnId)) {
2114                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2115                         vrfEntry, dpnId);
2116                 return false;
2117             }
2118         }
2119         return true;
2120     }
2121
2122     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2123         // FIXME: use vpnInstance.key() instead?
2124         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2125     }
2126
2127     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2128         return JvmGlobalLocks.getLockFor(label);
2129     }
2130 }