fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  12 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  13
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.FutureCallback;
  17 import com.google.common.util.concurrent.Futures;
  18 import com.google.common.util.concurrent.ListenableFuture;
  19 import com.google.common.util.concurrent.MoreExecutors;
  20 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  21 import java.math.BigInteger;
  22 import java.net.InetAddress;
  23 import java.net.UnknownHostException;
  24 import java.util.ArrayList;
  25 import java.util.Arrays;
  26 import java.util.Collections;
  27 import java.util.HashMap;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Optional;
  32 import java.util.concurrent.Callable;
  33 import java.util.concurrent.CopyOnWriteArrayList;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.locks.ReentrantLock;
  36 import javax.inject.Inject;
  37 import javax.inject.Singleton;
  38 import org.eclipse.jdt.annotation.NonNull;
  39 import org.eclipse.jdt.annotation.Nullable;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  47 import org.opendaylight.genius.infra.TransactionAdapter;
  48 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  49 import org.opendaylight.genius.infra.TypedWriteTransaction;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.FlowEntity;
  52 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  53 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  54 import org.opendaylight.genius.mdsalutil.MatchInfo;
  55 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  64 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  70 import org.opendaylight.genius.utils.JvmGlobalLocks;
  71 import org.opendaylight.genius.utils.ServiceIndex;
  72 import org.opendaylight.genius.utils.batching.SubTransaction;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.Executors;
  75 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  76 import org.opendaylight.mdsal.binding.api.DataBroker;
  77 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  78 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  79 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  80 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  81 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  82 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  83 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  85 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  86 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  87 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePathsKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 124 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 125 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 126 import org.opendaylight.yangtools.yang.common.Uint32;
 127 import org.opendaylight.yangtools.yang.common.Uint64;
 128 import org.slf4j.Logger;
 129 import org.slf4j.LoggerFactory;
 130
 131 @Singleton
 132 public class VrfEntryListener extends AbstractAsyncDataTreeChangeListener<VrfEntry> {
 133
 134     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 135     private static final String FLOWID_PREFIX = "L3.";
 136     private static final Uint64 COOKIE_VM_FIB_TABLE =  Uint64.valueOf("8000003", 16).intern();
 137     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 138     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 139     private static final int LFIB_INTERVPN_PRIORITY = 15;
 140     public static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 141     private static final int MAX_RETRIES = 3;
 142     private static final Uint64 COOKIE_TABLE_MISS = Uint64.valueOf("8000004", 16).intern();
 143
 144     private final DataBroker dataBroker;
 145     private final ManagedNewTransactionRunner txRunner;
 146     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 147     private final IMdsalApiManager mdsalManager;
 148     private final NexthopManager nextHopManager;
 149     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 150     private final BaseVrfEntryHandler baseVrfEntryHandler;
 151     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 152     private final JobCoordinator jobCoordinator;
 153     private final IElanService elanManager;
 154     private final FibUtil fibUtil;
 155     private final InterVpnLinkCache interVpnLinkCache;
 156     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 157     private final UpgradeState upgradeState;
 158     private final DataTreeEventCallbackRegistrar eventCallbacks;
 159
 160     @Inject
 161     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 162                             final NexthopManager nexthopManager,
 163                             final IElanService elanManager,
 164                             final BaseVrfEntryHandler vrfEntryHandler,
 165                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 166                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 167                             final JobCoordinator jobCoordinator,
 168                             final FibUtil fibUtil,
 169                             final InterVpnLinkCache interVpnLinkCache,
 170                             final UpgradeState upgradeState,
 171                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 172         super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(FibEntries.class)
 173                 .child(VrfTables.class).child(VrfEntry.class),
 174                 Executors.newListeningSingleThreadExecutor("VrfEntryListener", LOG));
 175         this.dataBroker = dataBroker;
 176         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 177         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 178         this.mdsalManager = mdsalApiManager;
 179         this.nextHopManager = nexthopManager;
 180         this.elanManager = elanManager;
 181         this.baseVrfEntryHandler = vrfEntryHandler;
 182         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 183         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 184         this.jobCoordinator = jobCoordinator;
 185         this.fibUtil = fibUtil;
 186         this.interVpnLinkCache = interVpnLinkCache;
 187         this.upgradeState = upgradeState;
 188         this.eventCallbacks = eventCallbacks;
 189     }
 190
 191     public void init() {
 192         LOG.info("{} init", getClass().getSimpleName());
 193     }
 194
 195     @Override
 196     @SuppressWarnings("checkstyle:IllegalCatch")
 197     public void close() {
 198         closeables.forEach(c -> {
 199             try {
 200                 c.close();
 201             } catch (Exception e) {
 202                 LOG.warn("Error closing {}", c, e);
 203             }
 204         });
 205         Executors.shutdownAndAwaitTermination(getExecutorService());
 206     }
 207
 208     @Override
 209     public void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 210         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 211         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 212         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 213                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 214         addFibEntries(identifier, vrfEntry, rd);
 215         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 216                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 217     }
 218
 219     //This method is temporary. Eventually Factory design pattern will be used to get
 220     // right VrfEntryhandle and invoke its methods.
 221     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 222         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 223             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 224             return;
 225         }
 226         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 227             LOG.info("EVPN flows need to be programmed.");
 228             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 229                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 230             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             closeables.add(evpnVrfEntryHandler);
 232             return;
 233         }
 234         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 235         if (routerInt != null) {
 236             // ping responder for router interfaces
 237             routerInterfaceVrfEntryHandler.createFlows(vrfEntry, rd);
 238             return;
 239         }
 240         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 241             createFibEntries(identifier, vrfEntry);
 242             return;
 243         }
 244     }
 245
 246     @Override
 247     public void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 248         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 249         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 250         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 251                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 252         removeFibEntries(identifier, vrfEntry, rd);
 253         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 254             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 255     }
 256
 257     //This method is temporary. Eventually Factory design pattern will be used to get
 258     // right VrfEntryhandle and invoke its methods.
 259     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 260         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 261             LOG.info("EVPN flows to be deleted");
 262             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 263                     nextHopManager, jobCoordinator, fibUtil, upgradeState, eventCallbacks);
 264             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 265             closeables.add(evpnVrfEntryHandler);
 266             return;
 267         }
 268         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 269         if (routerInt != null) {
 270             // ping responder for router interfaces
 271             routerInterfaceVrfEntryHandler.removeFlows(vrfEntry, rd);
 272             return;
 273         }
 274         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 275             deleteFibEntries(identifier, vrfEntry);
 276             return;
 277         }
 278         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 279             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 280             return;
 281         }
 282     }
 283
 284     @Override
 285     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 286     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 287     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 288     public void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 289         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 290         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 291         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 292                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 293         // Handle BGP Routes first
 294         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 295             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 296             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 297                     rd, update.getDestPrefix(), update.getRoutePaths());
 298             return;
 299         }
 300
 301         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 302             List<RoutePaths> originalRoutePath = new ArrayList<RoutePaths>(original.nonnullRoutePaths().values());
 303             List<RoutePaths> updateRoutePath = new ArrayList<RoutePaths>(update.nonnullRoutePaths().values());
 304             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 305
 306             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 307             //updated vrf entry route path is null. This can happen during tunnel events.
 308             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 309             List<String> usedRds = new ArrayList<>();
 310             if (optVpnInstance.isPresent()) {
 311                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 312                         update.getDestPrefix());
 313             }
 314             // If original VRF Entry had nexthop null , but update VRF Entry
 315             // has nexthop , route needs to be created on remote Dpns
 316             if (originalRoutePath == null || originalRoutePath.isEmpty()
 317                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 318                 // TODO(vivek): Though ugly, Not handling this code now, as each
 319                 // tep add event will invoke flow addition
 320                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 321                         + " This event is IGNORED here.", update.getDestPrefix());
 322                 return;
 323             }
 324
 325             // If original VRF Entry had valid nexthop , but update VRF Entry
 326             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 327             if (updateRoutePath == null || updateRoutePath.isEmpty()
 328                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 329                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 330                         + "This event is IGNORED here.", update.getDestPrefix());
 331                 return;
 332             }
 333             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 334             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 335             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 336             List<ListenableFuture<Void>> futures = new ArrayList<>();
 337             ListenableFuture<Void> configFuture =
 338                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 339                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 340                         nextHopsRemoved.parallelStream()
 341                             .forEach(nextHopRemoved -> {
 342                                 try {
 343                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 344                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 345                                 } catch (ExecutionException | InterruptedException e) {
 346                                     throw new RuntimeException(e);
 347                                 }
 348                             }))));
 349             futures.add(configFuture);
 350             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 351                 @Override
 352                 public void onSuccess(Void result) {
 353                     createFibEntries(identifier, update);
 354                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 355                             rd, update.getDestPrefix(), update.getRoutePaths());
 356                 }
 357
 358                 @Override
 359                 public void onFailure(Throwable throwable) {
 360                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 361                             update, throwable);
 362                 }
 363             }, MoreExecutors.directExecutor());
 364             return;
 365         }
 366
 367         //Handle all other routes only on a cluster reboot
 368         if (original.equals(update)) {
 369             //Reboot use-case
 370             createFibEntries(identifier, update);
 371             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 372                     rd, update.getDestPrefix(), update.getRoutePaths());
 373             return;
 374         }
 375
 376         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 377                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 378     }
 379
 380     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 381         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 382         List<SubTransaction> txnObjects =  new ArrayList<>();
 383         final VpnInstanceOpDataEntry vpnInstance =
 384                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 385         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 386         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 387                 + " has null vpnId!");
 388         final Map<VpnToDpnListKey, VpnToDpnList> keyVpnToDpnListMap;
 389         if (vrfEntry.getParentVpnRd() != null
 390                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 391             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 392             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 393             keyVpnToDpnListMap = parentVpnInstance != null ? parentVpnInstance.nonnullVpnToDpnList() :
 394                 vpnInstance.getVpnToDpnList();
 395             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 396                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 397         } else {
 398             keyVpnToDpnListMap = vpnInstance.nonnullVpnToDpnList();
 399         }
 400         final Uint32 vpnId = vpnInstance.getVpnId();
 401         final String rd = vrfTableKey.getRouteDistinguisher();
 402         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 403         if (subnetRoute != null) {
 404             final long elanTag = subnetRoute.getElantag().toJava();
 405             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 406                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 407             if (keyVpnToDpnListMap != null) {
 408                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 409                     () -> Collections.singletonList(
 410                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 411                             for (final VpnToDpnList curDpn : keyVpnToDpnListMap.values()) {
 412                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 413                                     installSubnetRouteInFib(curDpn.getDpnId(),
 414                                                                 elanTag, rd, vpnId, vrfEntry, tx);
 415                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
 416                                         vpnId, vrfEntry, NwConstants.ADD_FLOW, tx);
 417                                 }
 418                             }
 419                         })));
 420             }
 421             return;
 422         }
 423         // Get etherType value based on the IpPrefix address family type
 424         int etherType;
 425         try {
 426             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 427         } catch (IllegalArgumentException ex) {
 428             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 429             return;
 430         }
 431
 432         final List<Uint64> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(),
 433                                                                         rd, vrfEntry, etherType);
 434         if (!localDpnIdList.isEmpty() && keyVpnToDpnListMap != null) {
 435             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 436                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 437                     final ReentrantLock lock = lockFor(vpnInstance);
 438                     lock.lock();
 439                     try {
 440                         for (VpnToDpnList vpnDpn : keyVpnToDpnListMap.values()) {
 441                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 442                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 443                                     try {
 444                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 445                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 446                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 447                                                     TransactionAdapter.toWriteTransaction(tx), txnObjects);
 448                                         } else {
 449                                             createRemoteFibEntry(vpnDpn.getDpnId(),
 450                                                     vpnInstance.getVpnId(),
 451                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 452                                         }
 453                                     } catch (NullPointerException e) {
 454                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 455                                                 vrfEntry.getDestPrefix(), e);
 456                                     }
 457                                 }
 458                             }
 459                         }
 460                     } finally {
 461                         lock.unlock();
 462                     }
 463                 })), MAX_RETRIES);
 464         }
 465
 466         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 467         if (optVpnUuid.isPresent()) {
 468             String vpnUuid = optVpnUuid.get();
 469             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orElse(null);
 470             if (interVpnLink != null) {
 471                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 472                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 473                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 474                         // This is an static route that points to the other endpoint of an InterVpnLink
 475                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 476                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 477                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 478                     }
 479                 });
 480             }
 481         }
 482     }
 483
 484     void refreshFibTables(String rd, String prefix) {
 485         InstanceIdentifier<VrfEntry> vrfEntryId =
 486                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 487                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 488         Optional<VrfEntry> vrfEntry;
 489         try {
 490             vrfEntry = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 491                     vrfEntryId);
 492         } catch (ExecutionException | InterruptedException e) {
 493             LOG.error("refreshFibTables: Exception while reading VrfEntry Ds for the prefix {} rd {}", prefix, rd, e);
 494             return;
 495         }
 496         if (vrfEntry.isPresent()) {
 497             createFibEntries(vrfEntryId, vrfEntry.get());
 498         }
 499     }
 500
 501     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 502         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 503         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 504         prefixBuilder.setDpnId(lri.getDpnId());
 505         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 506         prefixBuilder.setIpAddress(lri.getPrefix());
 507         // Increment the refCount here
 508         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 509             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 510         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 511         if (!isPresentInList) {
 512             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 513             List<String> vpnInstanceNames =
 514                 lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
 515             vpnInstanceNames.add(vpnInstanceName);
 516             builder.setVpnInstanceList(vpnInstanceNames);
 517             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 518         } else {
 519             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 520         }
 521         return prefixBuilder.build();
 522     }
 523
 524     void installSubnetRouteInFib(final Uint64 dpnId, final long elanTag, final String rd,
 525             final Uint32 vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 526         if (tx == null) {
 527             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 528                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 529                 "Error installing subnet route in FIB");
 530             return;
 531         }
 532         int etherType;
 533         try {
 534             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 535         } catch (IllegalArgumentException ex) {
 536             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 537             return;
 538         }
 539         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 540             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 541             final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 542             final ReentrantLock lock = lockFor(lriKey);
 543             lock.lock();
 544             try {
 545                 LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 546                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 547
 548                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 549                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 550                                 fibUtil.getVpnInstanceOpData(rd);
 551                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 552                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 553                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 554                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 555                             }
 556                         }
 557                     }
 558                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 559                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 560                 }
 561             } finally {
 562                 lock.unlock();
 563             }
 564         });
 565         final List<InstructionInfo> instructions = new ArrayList<>();
 566         Uint64 subnetRouteMeta = Uint64.valueOf(BigInteger.valueOf(elanTag).shiftLeft(24)
 567             .or(BigInteger.valueOf(vpnId.longValue()).shiftLeft(1)));
 568         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 569         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 570         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 571                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 572         if (vrfEntry.getRoutePaths() != null) {
 573             for (RoutePaths routePath : vrfEntry.getRoutePaths().values()) {
 574                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 575                     List<ActionInfo> actionsInfos = new ArrayList<>();
 576                     // reinitialize instructions list for LFIB Table
 577                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 578                     actionsInfos.add(new ActionPopMpls(etherType));
 579                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 580                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 581                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 582                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 583
 584                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions,
 585                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 586                 }
 587             }
 588         }
 589     }
 590
 591     private void installSubnetBroadcastAddrDropRule(final Uint64 dpnId, final String rd, final Uint32 vpnId,
 592             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 593         List<MatchInfo> matches = new ArrayList<>();
 594
 595         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 596                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 597         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 598         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 599         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 600
 601         InetAddress destPrefix;
 602         try {
 603             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 604         } catch (UnknownHostException e) {
 605             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 606                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 607             return;
 608         }
 609
 610         // Match on VpnId and SubnetBroadCast IP address
 611         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 612             MetaDataUtil.METADATA_MASK_VRFID));
 613         matches.add(MatchEthernetType.IPV4);
 614
 615         if (prefixLength != 0) {
 616             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 617         }
 618
 619         //Action is to drop the packet
 620         List<InstructionInfo> dropInstructions = new ArrayList<>();
 621         List<ActionInfo> actionsInfos = new ArrayList<>();
 622         actionsInfos.add(new ActionDrop());
 623         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 624
 625         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 626         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 627         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 628                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 629
 630         Flow flow = flowEntity.getFlowBuilder().build();
 631         String flowId = flowEntity.getFlowId();
 632         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 633         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 634
 635         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 636                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 637                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 638
 639         if (addOrRemove == NwConstants.ADD_FLOW) {
 640             tx.mergeParentStructurePut(flowInstanceId,flow);
 641         } else {
 642             tx.delete(flowInstanceId);
 643         }
 644     }
 645
 646     /*
 647      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 648      * LportDispatcher table (via table 80)
 649      */
 650     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 651                                             final VrfEntry vrfEntry, int etherType) {
 652         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 653         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 654         // packet is commuted from Vpn2 to Vpn1.
 655         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 656         if (!interVpnLink.isActive()) {
 657             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 658                      interVpnLinkName, vrfEntry.getDestPrefix());
 659             return;
 660         }
 661
 662         Optional<Uint32> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 663         if (!optLportTag.isPresent()) {
 664             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 665             return;
 666         }
 667
 668         Long lportTag = optLportTag.get().toJava();
 669         Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 670         if (label == null) {
 671             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 672                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 673             return;
 674         }
 675         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 676         List<InstructionInfo> instructions = Arrays.asList(
 677             new InstructionApplyActions(actionsInfos),
 678             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 679                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 680                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 681                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 682             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 683         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 684         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 685
 686         for (Uint64 dpId : targetDpns) {
 687             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 688                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 689
 690             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 691                                /*writeTx*/null);
 692         }
 693     }
 694
 695
 696     /*
 697      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 698      */
 699     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 700                                                final VrfEntry vrfEntry, Uint32 vpnTag) {
 701         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 702         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 703             && vrfEntry.getRoutePaths().size() == 1);
 704         String destination = vrfEntry.getDestPrefix();
 705         String nextHop = new ArrayList<RoutePaths>(vrfEntry.getRoutePaths().values()).get(0).getNexthopAddress();
 706         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 707
 708         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 709         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 710         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 711         if (interVpnLink.getState().orElse(State.Error) != State.Active) {
 712             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 713                 destination, nextHop, interVpnLinkName);
 714             return;
 715         }
 716
 717         Optional<Uint32> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 718         if (!optOtherEndpointLportTag.isPresent()) {
 719             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 720                 vpnUuid, interVpnLinkName);
 721             return;
 722         }
 723
 724         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 725         if (targetDpns.isEmpty()) {
 726             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 727                 vpnUuid, interVpnLinkName);
 728             return;
 729         }
 730
 731         String[] values = destination.split("/");
 732         String destPrefixIpAddress = values[0];
 733         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 734
 735         List<MatchInfo> matches = new ArrayList<>();
 736         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag.longValue()),
 737             MetaDataUtil.METADATA_MASK_VRFID));
 738         matches.add(MatchEthernetType.IPV4);
 739
 740         if (prefixLength != 0) {
 741             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 742         }
 743
 744         List<Instruction> instructions =
 745             Arrays.asList(new InstructionWriteMetadata(
 746                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 747                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 748                             .L3VPN_SERVICE_INDEX)),
 749                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 750                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 751         Map<InstructionKey, Instruction> instructionsMap = new HashMap<InstructionKey, Instruction>();
 752         int instructionKey = 0;
 753         for (Instruction instructionObj : instructions) {
 754             instructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
 755         }
 756
 757         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 758         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 759         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 760             COOKIE_VM_FIB_TABLE, matches, instructionsMap);
 761
 762         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 763             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 764
 765         for (Uint64 dpId : targetDpns) {
 766
 767             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 768                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 769                 dpId, interVpnLink.getInterVpnLinkName());
 770
 771             mdsalManager.installFlow(dpId, flowEntity);
 772         }
 773     }
 774
 775     private List<Uint64> createLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 776         List<Uint64> returnLocalDpnId = new ArrayList<>();
 777         String localNextHopIP = vrfEntry.getDestPrefix();
 778         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 779         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 780         if (localNextHopInfo == null) {
 781             boolean localNextHopSeen = false;
 782             List<Routes> vpnExtraRoutes = null;
 783             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 784             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 785             // FIXME: use an Identifier structure?
 786             final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd);
 787             lock.lock();
 788             try {
 789                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 790                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 791                         vpnName, usedRds, localNextHopIP);
 792                 if (LOG.isDebugEnabled()) {
 793                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 794                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 795                 }
 796
 797                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 798                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 799                     String ipPrefix;
 800                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 801                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 802                     } else {
 803                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 804                     }
 805                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 806                             ipPrefix);
 807                     if (localNextHopInfoLocal != null) {
 808                         localNextHopSeen = true;
 809                         Uint64 dpnId =
 810                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 811                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType,
 812                                         /*parentVpnId*/ null);
 813                         returnLocalDpnId.add(dpnId);
 814                     }
 815                 }
 816             } finally {
 817                 lock.unlock();
 818             }
 819             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 820                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 821                 if (optionalLabel.isPresent()) {
 822                     Uint32 label = optionalLabel.get();
 823                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 824                     final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
 825                     final ReentrantLock labelLock = lockFor(lriKey);
 826                     labelLock.lock();
 827                     try {
 828                         LabelRouteInfo lri = getLabelRouteInfo(lriKey);
 829                         Uint32 parentVpnId = lri.getParentVpnid();
 830                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 831                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 832                                     fibUtil.getVpnInstanceOpData(rd);
 833                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 834                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 835                                 if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains(
 836                                        vpnInstanceName)) {
 837                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 838                                     localNextHopIP = lri.getPrefix();
 839                                 } else {
 840                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 841                                     localNextHopIP = lri.getPrefix();
 842                                 }
 843                             }
 844                             if (localNextHopInfo != null) {
 845                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 846                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 847                                 if (vpnExtraRoutes.isEmpty()) {
 848                                     Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 849                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType, parentVpnId);
 850                                     returnLocalDpnId.add(dpnId);
 851                                 } else {
 852                                     for (Routes extraRoutes : vpnExtraRoutes) {
 853                                         Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 854                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType,
 855                                                 parentVpnId);
 856                                         returnLocalDpnId.add(dpnId);
 857                                     }
 858                                 }
 859                             }
 860                         }
 861                     } finally {
 862                         labelLock.unlock();
 863                     }
 864                 }
 865             }
 866             if (returnLocalDpnId.isEmpty()) {
 867                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 868             }
 869         } else {
 870             Uint64 dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 871                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType,
 872                     /*parentVpnId*/ null);
 873             if (dpnId != null) {
 874                 returnLocalDpnId.add(dpnId);
 875             }
 876         }
 877         return returnLocalDpnId;
 878     }
 879
 880     private Uint64 checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 881                                                 final Uint32 vpnId, final String rd,
 882                                                 final VrfEntry vrfEntry,
 883                                                 @Nullable Routes routes, @Nullable List<Routes> vpnExtraRoutes,
 884                                                 int etherType, Uint32 parentVpnId) {
 885         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 886         if (localNextHopInfo != null) {
 887             long groupId;
 888             long localGroupId;
 889             final Uint64 dpnId = localNextHopInfo.getDpnId();
 890             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 891                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 892                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 893                 return dpnId;
 894             }
 895             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 896                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 897                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 898                 return dpnId;
 899             }
 900             if (!isVpnPresentInDpn(rd, dpnId)) {
 901                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 902                         vpnId, rd, dpnId.toString());
 903                 return Uint64.ZERO;
 904             }
 905             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 906             String prefix = vrfEntry.getDestPrefix();
 907             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 908             //The loadbalancing group is created only if the extra route has multiple nexthops
 909             //to avoid loadbalancing the discovered routes
 910             if (RouteOrigin.STATIC.getValue().equals(vrfEntry.getOrigin()) && vpnExtraRoutes != null
 911                     && routes != null) {
 912                 if (vpnExtraRoutes.size() > 1) {
 913                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 914                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 915                 } else {
 916                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 917                     localGroupId = groupId;
 918                 }
 919             } else {
 920                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 921                         gwMacAddress, parentVpnId);
 922                 localGroupId = groupId;
 923             }
 924             if (groupId == FibConstants.INVALID_GROUP_ID) {
 925                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 926                         prefix, rd, interfaceName, dpnId.toString());
 927                 return Uint64.ZERO;
 928             }
 929             final List<InstructionInfo> instructions = Collections.singletonList(
 930                     new InstructionApplyActions(
 931                             Collections.singletonList(new ActionGroup(groupId))));
 932             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 933                     new InstructionApplyActions(
 934                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 935             java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 936             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 937             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 938             jobCoordinator.enqueueJob(jobKey,
 939                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 940                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 941                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 942                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 943                         optLabel.ifPresent(label -> {
 944                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 945                                 LOG.debug(
 946                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 947                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 948                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 949                                         nextHopAddressList);
 950                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 951                                         NwConstants.ADD_FLOW, tx);
 952                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 953                             } else {
 954                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 955                                                 + "route. LFib and Terminating table entries will not be created.",
 956                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 957                             }
 958                         });
 959                     }
 960                 })));
 961             return dpnId;
 962         }
 963         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 964                 vpnName);
 965         return Uint64.ZERO;
 966     }
 967
 968     private boolean isVpnPresentInDpn(String rd, Uint64 dpnId) {
 969         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 970         Optional<VpnToDpnList> dpnInVpn;
 971         try {
 972             dpnInVpn = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 973         } catch (ExecutionException | InterruptedException e) {
 974             LOG.error("isVpnPresentInDpn: Exception while reading VpnToDpnList Ds for the rd {} dpnId {}", rd,
 975                     dpnId, e);
 976             return false;
 977         }
 978         return dpnInVpn.isPresent();
 979     }
 980
 981     @Nullable
 982     private LabelRouteInfo getLabelRouteInfo(Uint32 label) {
 983         return getLabelRouteInfo(new LabelRouteInfoKey(label));
 984     }
 985
 986     @Nullable
 987     private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) {
 988         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 989             .child(LabelRouteInfo.class, label).build();
 990         Optional<LabelRouteInfo> opResult = null;
 991         try {
 992             opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 993                     lriIid);
 994         } catch (ExecutionException | InterruptedException e) {
 995             LOG.error("refreshFibTables: Exception while reading LabelRouteInfo Ds for the label {}", label, e);
 996             return null;
 997         }
 998         if (opResult.isPresent()) {
 999             return opResult.get();
1000         }
1001         return null;
1002     }
1003
1004     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
1005             @Nullable TypedWriteTransaction<Operational> tx) {
1006         if (lri == null) {
1007             return true;
1008         }
1009
1010         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
1011         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
1012             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
1013
1014         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
1015             ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>();
1016         if (vpnInstancesList.contains(vpnInstanceName)) {
1017             LOG.debug("vpninstance {} name is present", vpnInstanceName);
1018             vpnInstancesList.remove(vpnInstanceName);
1019         }
1020         if (vpnInstancesList.isEmpty()) {
1021             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
1022             if (tx != null) {
1023                 tx.delete(lriId);
1024             } else {
1025                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
1026             }
1027             return true;
1028         } else {
1029             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
1030             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
1031             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
1032         }
1033         return false;
1034     }
1035
1036     void makeTunnelTableEntry(Uint64 dpId, Uint32 label, long groupId/*String egressInterfaceName*/,
1037                                       TypedWriteTransaction<Configuration> tx) {
1038         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
1039
1040         createTerminatingServiceActions(dpId, label, actionsInfos, tx);
1041
1042         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
1043             dpId, label, groupId);
1044     }
1045
1046     public void createTerminatingServiceActions(Uint64 destDpId, Uint32 label, List<ActionInfo> actionsInfos,
1047                                                 TypedWriteTransaction<Configuration> tx) {
1048         List<MatchInfo> mkMatches = new ArrayList<>();
1049
1050         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
1051             destDpId, label, actionsInfos);
1052
1053         // Matching metadata
1054         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1055         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1056
1057         List<InstructionInfo> mkInstructions = new ArrayList<>();
1058         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1059
1060         FlowEntity terminatingServiceTableFlowEntity =
1061             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1062             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1063                     FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1064                     String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1065                     Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()),
1066                     mkMatches, mkInstructions);
1067
1068         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1069
1070         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1071
1072         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1073         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1074             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1075             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1076             .child(Flow.class, flowKey).build();
1077         tx.mergeParentStructurePut(flowInstanceId, flowbld.build());
1078     }
1079
1080     private void removeTunnelTableEntry(Uint64 dpId, Uint32 label, TypedWriteTransaction<Configuration> tx) {
1081         FlowEntity flowEntity;
1082         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1083         List<MatchInfo> mkMatches = new ArrayList<>();
1084         // Matching metadata
1085         mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue())));
1086         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1087             NwConstants.INTERNAL_TUNNEL_TABLE,
1088             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, label),
1089                 FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1090                 String.format("%s:%s", "TST Flow Entry ", label), 0, 0,
1091                 Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()), mkMatches, null);
1092         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1093         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1094         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1095             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1096             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1097
1098         tx.delete(flowInstanceId);
1099         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1100     }
1101
1102     public List<Uint64> deleteLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry) {
1103         List<Uint64> returnLocalDpnId = new ArrayList<>();
1104         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1105         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1106         boolean shouldUpdateNonEcmpLocalNextHop = true;
1107         if (localNextHopInfo == null) {
1108             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1109             if (usedRds.size() > 1) {
1110                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1111                         vrfEntry.getDestPrefix(), vpnName, rd);
1112                 return returnLocalDpnId;
1113             }
1114             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1115             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1116             //in the vpn
1117             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1118                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1119             if (extraRouteOptional.isPresent()) {
1120                 Routes extraRoute = extraRouteOptional.get();
1121                 String ipPrefix;
1122                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1123                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1124                 } else {
1125                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1126                 }
1127                 if (extraRoute.getNexthopIpList().size() > 1) {
1128                     shouldUpdateNonEcmpLocalNextHop = false;
1129                 }
1130                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1131                 if (localNextHopInfo != null) {
1132                     String localNextHopIP = localNextHopInfo.getIpAddress();
1133                     Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1134                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1135                     if (!dpnId.equals(Uint64.ZERO)) {
1136                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1137                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1138                         returnLocalDpnId.add(dpnId);
1139                     }
1140                 } else {
1141                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1142                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1143                 }
1144             }
1145
1146             if (localNextHopInfo == null) {
1147                 /* Imported VRF entry */
1148                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1149                 if (optionalLabel.isPresent()) {
1150                     Uint32 label = optionalLabel.get();
1151                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1152                     LabelRouteInfo lri = getLabelRouteInfo(label);
1153                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1154                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1155                         prefixBuilder.setDpnId(lri.getDpnId());
1156                         Uint64 dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1157                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1158                         if (!dpnId.equals(Uint64.ZERO)) {
1159                             returnLocalDpnId.add(dpnId);
1160                         }
1161                     }
1162                 }
1163             }
1164
1165         } else {
1166             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1167             String localNextHopIP = localNextHopInfo.getIpAddress();
1168             Uint64 dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1169                     shouldUpdateNonEcmpLocalNextHop);
1170             if (!dpnId.equals(Uint64.ZERO)) {
1171                 returnLocalDpnId.add(dpnId);
1172             }
1173         }
1174
1175         return returnLocalDpnId;
1176     }
1177
1178     private Uint64 checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1179             final String vpnName, final Uint32 vpnId, final String rd, final VrfEntry vrfEntry,
1180             boolean shouldUpdateNonEcmpLocalNextHop) {
1181         if (localNextHopInfo != null) {
1182             final Uint64 dpnId = localNextHopInfo.getDpnId();
1183             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1184                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1185                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1186                 return dpnId;
1187             }
1188             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1189                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1190                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1191                 return dpnId;
1192             }
1193
1194             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1195                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1196                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1197                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1198                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1199                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1200                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1201                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1202                                         NwConstants.DEL_FLOW, tx);
1203                                 removeTunnelTableEntry(dpnId, label, tx);
1204                             });
1205                         }
1206                     }
1207                 })));
1208             //TODO: verify below adjacency call need to be optimized (?)
1209             //In case of the removal of the extra route, the loadbalancing group is updated
1210             if (shouldUpdateNonEcmpLocalNextHop) {
1211                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1212             }
1213             return dpnId;
1214         }
1215         return Uint64.ZERO;
1216     }
1217
1218     private void createRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1219             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1220         if (tx == null) {
1221             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1222                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1223                 "Error creating remote FIB entry");
1224             return;
1225         }
1226
1227         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1228         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", vrfEntry.getDestPrefix(), rd,
1229                 remoteDpnId);
1230
1231         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
1232             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1233             return;
1234         }
1235         // Handling static VRF entries
1236         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1237         List<Routes> vpnExtraRoutes =
1238                 VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, vpnName, usedRds, vrfEntry.getDestPrefix());
1239         if (!vpnExtraRoutes.isEmpty()) {
1240             programRemoteFibWithLoadBalancingGroups(remoteDpnId, vpnId, rd, vrfEntry, vpnExtraRoutes);
1241         } else {
1242             // Program in case of other static VRF entries like floating IPs
1243             programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx);
1244         }
1245     }
1246
1247     // Allow deprecated TransactionRunner calls for now
1248     @SuppressWarnings("ForbidCertainMethod")
1249     private void programRemoteFibWithLoadBalancingGroups(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1250             final VrfEntry vrfEntry, List<Routes> vpnExtraRoutes) {
1251         // create loadbalancing groups for extra routes only when the extra route is
1252         // present behind multiple VMs
1253         // Obtain the local routes for this particular dpn.
1254         java.util.Optional<Routes> routes = vpnExtraRoutes.stream().filter(route -> {
1255             Prefixes prefixToInterface =
1256                     fibUtil.getPrefixToInterface(vpnId, FibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1257             if (prefixToInterface == null) {
1258                 return false;
1259             }
1260             return remoteDpnId.equals(prefixToInterface.getDpnId());
1261         }).findFirst();
1262         long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1263                 routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1264         if (groupId == FibConstants.INVALID_GROUP_ID) {
1265             LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", vrfEntry.getDestPrefix(), rd,
1266                     remoteDpnId);
1267             return;
1268         }
1269         List<ActionInfo> actionInfos = Collections.singletonList(new ActionGroup(groupId));
1270         List<InstructionInfo> instructions = Lists.newArrayList(new InstructionApplyActions(actionInfos));
1271         String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
1272         jobCoordinator.enqueueJob(jobKey,
1273             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> {
1274                 baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1275                         NwConstants.ADD_FLOW, txn, null);
1276             })));
1277
1278         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1279     }
1280
1281     private void programRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd,
1282             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1283         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1284         if (adjacencyResults.isEmpty()) {
1285             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", vrfEntry.getRoutePaths(), rd,
1286                     remoteDpnId);
1287             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1288             return;
1289         }
1290         baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, TransactionAdapter.toWriteTransaction(tx),
1291                 rd, adjacencyResults, null);
1292         LOG.debug("Successfully programmed FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1293     }
1294
1295     protected void cleanUpOpDataForFib(Uint32 vpnId, String primaryRd, final VrfEntry vrfEntry) {
1296     /* Get interface info from prefix to interface mapping;
1297         Use the interface info to get the corresponding vpn interface op DS entry,
1298         remove the adjacency corresponding to this fib entry.
1299         If adjacency removed is the last adjacency, clean up the following:
1300          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1301          - prefix to interface entry
1302          - vpn interface op DS
1303      */
1304         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1305         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1306         if (prefixInfo == null) {
1307             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1308             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1309             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1310             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
1311                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1312                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1313                     if (nextHopIp != null) {
1314                         String ipPrefix;
1315                         if (isIpv4Address(nextHopIp)) {
1316                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1317                         } else {
1318                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1319                         }
1320                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1321                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1322                     }
1323                 }
1324             }
1325             if (prefixInfo == null) {
1326                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1327                 if (optionalLabel.isPresent()) {
1328                     Uint32 label = optionalLabel.get();
1329                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1330                     LabelRouteInfo lri = getLabelRouteInfo(label);
1331                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1332                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1333                         prefixBuilder.setDpnId(lri.getDpnId());
1334                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1335                         prefixBuilder.setIpAddress(lri.getPrefix());
1336                         prefixInfo = prefixBuilder.build();
1337                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1338                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1339                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1340                     }
1341                 }
1342             }
1343         } else {
1344             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1345         }
1346     }
1347
1348     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1349                                           final VrfEntry vrfEntry, @Nullable final Routes extraRoute) {
1350
1351         if (prefixInfo == null) {
1352             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1353                     vrfEntry.getDestPrefix(), vpnId, rd);
1354             return; //Don't have any info for this prefix (shouldn't happen); need to return
1355         }
1356
1357         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1358             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1359                     vrfEntry.getDestPrefix(), vpnId, rd);
1360             return;
1361         }
1362
1363         String ifName = prefixInfo.getVpnInterfaceName();
1364         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1365             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1366     }
1367
1368     private class CleanupVpnInterfaceWorker implements Callable<List<? extends ListenableFuture<?>>> {
1369         Prefixes prefixInfo;
1370         Uint32 vpnId;
1371         String rd;
1372         VrfEntry vrfEntry;
1373         Routes extraRoute;
1374
1375         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Uint32 vpnId, final String rd,
1376                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1377             this.prefixInfo = prefixInfo;
1378             this.vpnId = vpnId;
1379             this.rd = rd;
1380             this.vrfEntry = vrfEntry;
1381             this.extraRoute = extraRoute;
1382         }
1383
1384         @Override
1385         public List<ListenableFuture<Void>> call() {
1386             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1387             // to call the respective helpers.
1388             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1389                 //First Cleanup LabelRouteInfo
1390                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1391                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1392                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1393                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1394                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1395                         final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1396                         final ReentrantLock lock = lockFor(lriKey);
1397                         lock.lock();
1398                         try {
1399                             LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1400                             if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix())
1401                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1402                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1403                                         fibUtil.getVpnInstanceOpData(rd);
1404                                 String vpnInstanceName = "";
1405                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1406                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1407                                 }
1408                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1409                                 if (lriRemoved) {
1410                                     String parentRd = lri.getParentVpnRd();
1411                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1412                                             parentRd, vrfEntry.getDestPrefix()));
1413                                 }
1414                             } else {
1415                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1416                                         rd, vrfEntry.getDestPrefix()));
1417                             }
1418                         } finally {
1419                             lock.unlock();
1420                         }
1421                     });
1422                 }
1423                 String ifName = prefixInfo.getVpnInterfaceName();
1424                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1425                 String vpnName = null;
1426
1427                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1428                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1429                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1430                     if (vpnNameOpt.isPresent()) {
1431                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1432                     }
1433                 }
1434                 if (optVpnName.isPresent()) {
1435                     vpnName = optVpnName.get();
1436                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1437                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1438                     if (opVpnInterface.isPresent()) {
1439                         Uint32 associatedVpnId = fibUtil.getVpnId(vpnName);
1440                         if (!Objects.equals(vpnId, associatedVpnId)) {
1441                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1442                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1443                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1444                             return;
1445                         } else {
1446                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1447                                     vrfEntry.getDestPrefix(), associatedVpnId);
1448                         }
1449                     }
1450                 }
1451                 if (extraRoute != null) {
1452                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1453                     //Only one used Rd present in case of removal event
1454                     String usedRd = usedRds.get(0);
1455                     if (optVpnName.isPresent()) {
1456                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1457                                         vrfEntry.getDestPrefix()));
1458                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1459                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1460                     }
1461                 }
1462                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1463             }));
1464         }
1465     }
1466
1467     /**
1468      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1469      * Remove Adjacency from VPNInterfaceOpData.
1470      * if Adjacency != primary.
1471      * if Adjacency == primary , then mark it for deletion.
1472      * Remove entire VPNinterfaceOpData Entry.
1473      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1474      * @param vrfEntry - VrfEntry removed
1475      * @param ifName - Interface name from VRFentry
1476      * @param vpnName - VPN name of corresponding VRF
1477      * @param tx - ReadWrite Tx
1478      */
1479     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
1480             justification = "https://github.com/spotbugs/spotbugs/issues/811")
1481     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1482                                                           TypedReadWriteTransaction<Operational> tx)
1483             throws ExecutionException, InterruptedException {
1484         InstanceIdentifier<Adjacency> adjacencyIid =
1485                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1486         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1487         if (adjacencyOptional.isPresent()) {
1488             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1489                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1490             } else {
1491                 tx.merge(adjacencyIid,
1492                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1493             }
1494         }
1495
1496         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1497
1498         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1499             return;
1500         }
1501
1502         @NonNull List<Adjacency> adjacencies
1503                 = new ArrayList<Adjacency>(optAdjacencies.get().nonnullAdjacency().values());
1504         if (adjacencies.size() <= 2
1505                 && adjacencies.stream().allMatch(adjacency ->
1506                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1507                         && adjacency.isMarkedForDeletion() != null
1508                         && adjacency.isMarkedForDeletion()
1509         )) {
1510             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1511             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1512         }
1513     }
1514
1515     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1516         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1517         final String rd = vrfTableKey.getRouteDistinguisher();
1518         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1519         if (vpnInstance == null) {
1520             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1521             return;
1522         }
1523         final Map<VpnToDpnListKey, VpnToDpnList> keyVpnToDpnListMap;
1524         if (vrfEntry.getParentVpnRd() != null
1525                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1526             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1527             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1528             keyVpnToDpnListMap = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1529                     vpnInstance.getVpnToDpnList();
1530             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1531                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1532         } else {
1533             keyVpnToDpnListMap = vpnInstance.getVpnToDpnList();
1534         }
1535
1536         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1537         final java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1538         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1539         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1540         if (subnetRoute != null) {
1541             long elanTag = subnetRoute.getElantag().toJava();
1542             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1543                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1544             if (keyVpnToDpnListMap != null) {
1545                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1546                     () -> Collections.singletonList(
1547                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1548                             for (final VpnToDpnList curDpn : keyVpnToDpnListMap.values()) {
1549
1550                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(),
1551                                     vpnInstance.getVpnId(),
1552                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null,
1553                                         NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1554                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1555                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(),
1556                                         label, null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1557                                 }
1558
1559                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd,
1560                                     vpnInstance.getVpnId(),
1561                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1562                             }
1563                         })));
1564             }
1565             optionalLabel.ifPresent(label -> {
1566                 final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label);
1567                 final ReentrantLock lock = lockFor(lriKey);
1568                 lock.lock();
1569                 try {
1570                     LabelRouteInfo lri = getLabelRouteInfo(lriKey);
1571                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1572                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1573                                 fibUtil.getVpnInstanceOpData(rd);
1574                         String vpnInstanceName = "";
1575                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1576                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1577                         }
1578                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1579                         if (lriRemoved) {
1580                             String parentRd = lri.getParentVpnRd();
1581                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1582                                     parentRd, vrfEntry.getDestPrefix()));
1583                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1584                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1585                         }
1586                     } else {
1587                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1588                                 rd, vrfEntry.getDestPrefix()));
1589                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1590                                 label, rd, vrfEntry.getDestPrefix());
1591                     }
1592                 } finally {
1593                     lock.unlock();
1594                 }
1595             });
1596             return;
1597         }
1598
1599         final List<Uint64> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1600             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1601         if (keyVpnToDpnListMap != null) {
1602             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1603                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1604             String jobKey;
1605             Optional<Routes> extraRouteOptional;
1606             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1607             if (usedRds != null && !usedRds.isEmpty()) {
1608                 if (usedRds.size() > 1) {
1609                     LOG.error("The extra route prefix is still present in some DPNs");
1610                     return ;
1611                 } else {
1612                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1613                     //is not present in any other DPN
1614                     extraRouteOptional = VpnExtraRouteHelper
1615                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1616                 }
1617             } else {
1618                 extraRouteOptional = Optional.empty();
1619             }
1620
1621             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1622                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1623                     if (localDpnIdList.size() <= 0) {
1624                         for (VpnToDpnList curDpn : keyVpnToDpnListMap.values()) {
1625                             baseVrfEntryHandler.deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(),
1626                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1627                                     TransactionAdapter.toWriteTransaction(tx));
1628                         }
1629                     } else {
1630                         for (Uint64 localDpnId : localDpnIdList) {
1631                             for (VpnToDpnList curDpn : keyVpnToDpnListMap.values()) {
1632                                 if (!Objects.equals(curDpn.getDpnId(), localDpnId)) {
1633                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1634                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1635                                             TransactionAdapter.toWriteTransaction(tx));
1636                                 }
1637                             }
1638                         }
1639                     }
1640                     if (extraRouteOptional.isPresent()) {
1641                         //Remove select groups only for extra-routes
1642                         nextHopManager.removeNextHopPointer(nextHopManager
1643                                 .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1644                         nextHopManager.removeNextHopPointer(nextHopManager
1645                                 .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1646                     }
1647                 })), MAX_RETRIES);
1648         }
1649
1650         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1651         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1652         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1653
1654         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1655         // of the interVpnLink.
1656         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1657         if (optVpnUuid.isPresent()) {
1658             String vpnUuid = optVpnUuid.get();
1659             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1660                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1661                 if (optInterVpnLink.isPresent()) {
1662                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1663                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1664                         // This is route that points to the other endpoint of an InterVpnLink
1665                         // In that case, we should look for the FIB table pointing to
1666                         // LPortDispatcher table and remove it.
1667                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1668                     }
1669                 }
1670             });
1671         }
1672
1673     }
1674
1675     private void makeLFibTableEntry(Uint64 dpId, Uint32 label, @Nullable List<InstructionInfo> instructions,
1676                                     int priority, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1677         if (tx == null) {
1678             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1679                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1680                 "Error making LFIB table entry");
1681             return;
1682         }
1683
1684         List<MatchInfo> matches = new ArrayList<>();
1685         matches.add(MatchEthernetType.MPLS_UNICAST);
1686         matches.add(new MatchMplsLabel(label.longValue()));
1687
1688         // Install the flow entry in L3_LFIB_TABLE
1689         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1690
1691         FlowEntity flowEntity;
1692         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1693             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1694         Flow flow = flowEntity.getFlowBuilder().build();
1695         String flowId = flowEntity.getFlowId();
1696         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1697         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1698         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1699             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1700             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1701
1702         if (addOrRemove == NwConstants.ADD_FLOW) {
1703             tx.mergeParentStructurePut(flowInstanceId, flow);
1704         } else {
1705             tx.delete(flowInstanceId);
1706         }
1707
1708         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1709             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1710     }
1711
1712     public void populateFibOnNewDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1713                                     final FutureCallback<List<Void>> callback) {
1714         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1715         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1716             () -> {
1717                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1718                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1719                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1720                         LogicalDatastoreType.CONFIGURATION, id);
1721                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1722                 if (!vrfTable.isPresent()) {
1723                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1724                     if (callback != null) {
1725                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1726                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1727                     }
1728                     return futures;
1729                 }
1730
1731                 final ReentrantLock lock = lockFor(vpnInstance);
1732                 lock.lock();
1733                 try {
1734                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1735                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry().values()) {
1736                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1737                             if (subnetRoute != null) {
1738                                 long elanTag = subnetRoute.getElantag().toJava();
1739                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1740                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1741                                         tx);
1742                                 continue;
1743                             }
1744                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1745                             if (routerInt != null) {
1746                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1747                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1748                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1749                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1750                                         NwConstants.ADD_FLOW);
1751                                 continue;
1752                             }
1753                             //Handle local flow creation for imports
1754                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1755                                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1756                                 if (optionalLabel.isPresent()) {
1757                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1758                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1759                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1760                                         if (Objects.equals(lri.getDpnId(), dpnId)) {
1761                                             try {
1762                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1763                                                         vrfEntry.getDestPrefix());
1764                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1765                                             } catch (IllegalArgumentException ex) {
1766                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1767                                                         vrfEntry.getDestPrefix());
1768                                             }
1769                                             continue;
1770                                         }
1771                                     }
1772                                 }
1773                             }
1774                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1775                                     vrfEntry, dpnId);
1776                             if (shouldCreateRemoteFibEntry) {
1777                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1778                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1779                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1780                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1781                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1782                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1783                                 } else {
1784                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1785                                             vrfEntry, tx);
1786                                 }
1787                             }
1788                         }
1789                     }));
1790                     if (callback != null) {
1791                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1792                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1793                     }
1794                 } finally {
1795                     lock.unlock();
1796                 }
1797                 return futures;
1798             });
1799     }
1800
1801     public void populateExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1802                                             final String localNextHopIp, final String remoteNextHopIp) {
1803         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1804             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1805         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1806         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1807         List<SubTransaction> txnObjects =  new ArrayList<>();
1808         final Optional<VrfTables> vrfTable;
1809         try {
1810             vrfTable = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1811         } catch (ExecutionException | InterruptedException e) {
1812             LOG.error("populateExternalRoutesOnDpn: Exception while reading the VrfTable for the rd {}", rd, e);
1813             return;
1814         }
1815         if (vrfTable.isPresent()) {
1816             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1817                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1818                     final ReentrantLock lock = lockFor(vpnInstance);
1819                     lock.lock();
1820                     try {
1821                         vrfTable.get().nonnullVrfEntry().values().stream()
1822                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1823                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1824                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1825                     } finally {
1826                         lock.unlock();
1827                     }
1828                 })));
1829         }
1830     }
1831
1832     public void manageRemoteRouteOnDPN(final boolean action,
1833                                        final Uint64 localDpnId,
1834                                        final Uint32 vpnId,
1835                                        final String rd,
1836                                        final String destPrefix,
1837                                        final String destTepIp,
1838                                        final Uint32 label) {
1839         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1840
1841         if (vpnInstance == null) {
1842             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1843             return;
1844         }
1845
1846         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1847             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1848                 final ReentrantLock lock = lockFor(vpnInstance);
1849                 lock.lock();
1850                 try {
1851                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1852                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1853                     if (vrfEntry == null) {
1854                         return;
1855                     }
1856                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1857                             action, localDpnId, vpnId, rd, destPrefix);
1858                     Map<RoutePathsKey, RoutePaths> keyRoutePathsMap = vrfEntry.getRoutePaths();
1859                     VrfEntry modVrfEntry;
1860                     if (keyRoutePathsMap == null || keyRoutePathsMap.isEmpty()) {
1861                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1862                                 Collections.singletonList(destTepIp),
1863                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1864                     } else {
1865                         modVrfEntry = vrfEntry;
1866                     }
1867
1868                     if (action) {
1869                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1870                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1871                                 modVrfEntry, tx);
1872                     } else {
1873                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1874                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1875                                 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1876                         if (usedRds.size() > 1) {
1877                             LOG.debug("The extra route prefix is still present in some DPNs");
1878                             return;
1879                         }
1880                         //Is this fib route an extra route? If yes, get the nexthop which would be
1881                         //an adjacency in the vpn
1882                         Optional<Routes> extraRouteOptional = Optional.empty();
1883                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC && usedRds.size() != 0) {
1884                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1885                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1886                                     usedRds.get(0), vrfEntry.getDestPrefix());
1887                         }
1888                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1889                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1890                     }
1891                 } finally {
1892                     lock.unlock();
1893                 }
1894             })));
1895     }
1896
1897     public void cleanUpDpnForVpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
1898                                  final FutureCallback<List<Void>> callback) {
1899         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1900         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1901             () -> {
1902                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1903                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1904                 List<SubTransaction> txnObjects = new ArrayList<>();
1905                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1906                         LogicalDatastoreType.CONFIGURATION, id);
1907                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1908                 if (!vrfTable.isPresent()) {
1909                     LOG.error("cleanUpDpnForVpn: VRF Table not available for RD {}", rd);
1910                     if (callback != null) {
1911                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1912                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1913                     }
1914                     return futures;
1915                 }
1916                 final ReentrantLock lock = lockFor(vpnInstance);
1917                 lock.lock();
1918                 try {
1919                     futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1920                         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1921                         for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry().values()) {
1922                             /* parentRd is only filled for external PNF cases where the interface on the external
1923                              * network VPN are used to cleanup the flows. For all other cases, use "rd" for
1924                              * #fibUtil.isInterfacePresentInDpn().
1925                              * */
1926                             String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd()
1927                                     : rd;
1928                             /* Handle subnet routes here */
1929                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1930                             if (subnetRoute != null && !fibUtil
1931                                     .isInterfacePresentInDpn(parentRd, dpnId)) {
1932                                 LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1933                                         + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1934                                 baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1935                                         NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1936                                 Map<RoutePathsKey, RoutePaths> keyRoutePathsMap = vrfEntry.getRoutePaths();
1937                                 if (keyRoutePathsMap != null) {
1938                                     for (RoutePaths routePath : keyRoutePathsMap.values()) {
1939                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1940                                                 DEFAULT_FIB_FLOW_PRIORITY,
1941                                                 NwConstants.DEL_FLOW, tx);
1942                                         LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1943                                                         + " for rd {} prefix {}", routePath.getLabel(), rd,
1944                                                 vrfEntry.getDestPrefix());
1945                                     }
1946                                 }
1947                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1948                                         NwConstants.DEL_FLOW, tx);
1949                                 continue;
1950                             }
1951                             // ping responder for router interfaces
1952                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1953                             if (routerInt != null) {
1954                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1955                                         rd, routerInt.getUuid(), routerInt.getIpAddress(),
1956                                         routerInt.getMacAddress());
1957                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1958                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1959                                         NwConstants.DEL_FLOW);
1960                                 continue;
1961                             }
1962                             //Handle local flow deletion for imports
1963                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1964                                 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1965                                 if (optionalLabel.isPresent()) {
1966                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1967                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1968                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1969                                             lri) && Objects.equals(lri.getDpnId(), dpnId)) {
1970                                         deleteLocalFibEntry(vpnId, rd, vrfEntry);
1971                                     }
1972                                 }
1973                             }
1974                             // Passing null as we don't know the dpn
1975                             // to which prefix is attached at this point
1976                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1977                                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1978                             Optional<Routes> extraRouteOptional;
1979                             //Is this fib route an extra route? If yes, get the nexthop which would be
1980                             //an adjacency in the vpn
1981                             if (usedRds != null && !usedRds.isEmpty()) {
1982                                 if (usedRds.size() > 1) {
1983                                     LOG.error("The extra route prefix is still present in some DPNs");
1984                                     return;
1985                                 } else {
1986                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1987                                             usedRds.get(0), vrfEntry.getDestPrefix());
1988
1989                                 }
1990                             } else {
1991                                 extraRouteOptional = Optional.empty();
1992                             }
1993                             if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1994                                 bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1995                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1996                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1997                             } else {
1998                                 if (subnetRoute == null || !fibUtil
1999                                         .isInterfacePresentInDpn(parentRd, dpnId)) {
2000                                     baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
2001                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
2002                                             TransactionAdapter.toWriteTransaction(tx));
2003                                 }
2004                             }
2005                         }
2006                     }));
2007                 } finally {
2008                     lock.unlock();
2009                 }
2010                 if (callback != null) {
2011                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
2012                     Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
2013                 }
2014                 return futures;
2015             });
2016     }
2017
2018     public void cleanUpExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd,
2019                                            final String localNextHopIp, final String remoteNextHopIp) {
2020         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2021                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2022             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2023         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2024         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
2025         List<SubTransaction> txnObjects =  new ArrayList<>();
2026         final Optional<VrfTables> vrfTable;
2027         try {
2028             vrfTable = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2029         } catch (ExecutionException | InterruptedException e) {
2030             LOG.error("getVrfEntry: Exception while reading VrfTable for the rd {} vpnId {}", rd, vpnId, e);
2031             return;
2032         }
2033         if (vrfTable.isPresent()) {
2034             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
2035                 () -> {
2036                     final ReentrantLock lock = lockFor(vpnInstance);
2037                     lock.lock();
2038                     try {
2039                         return Collections.singletonList(
2040                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
2041                                 tx -> vrfTable.get().nonnullVrfEntry().values().stream()
2042                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2043                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
2044                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
2045                                             txnObjects))));
2046                     } finally {
2047                         lock.unlock();
2048                     }
2049                 });
2050         }
2051     }
2052
2053     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2054         InstanceIdentifierBuilder<VrfTables> idBuilder =
2055             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2056         return idBuilder.build();
2057     }
2058
2059     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2060         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2061                 .FLOWID_SEPARATOR + nextHop;
2062     }
2063
2064     private String getTableMissFlowRef(Uint64 dpnId, short tableId, Uint32 tableMiss) {
2065         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2066                 + tableMiss + FLOWID_PREFIX;
2067     }
2068
2069     @Nullable
2070     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2071         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2072             .child(VrfTables.class, new VrfTablesKey(rd))
2073             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2074         Optional<VrfEntry> vrfEntry;
2075         try {
2076             vrfEntry = SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION,
2077                     vrfEntryId);
2078         } catch (ExecutionException | InterruptedException e) {
2079             LOG.error("getVrfEntry: Exception while reading VrfEntry for the prefix {} rd {}", ipPrefix, rd, e);
2080             return null;
2081         }
2082         if (vrfEntry.isPresent()) {
2083             return vrfEntry.get();
2084         }
2085         return null;
2086     }
2087
2088     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
2089                                              final String vpnName,
2090                                              final VrfEntry vrfEntry) {
2091         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
2092
2093         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
2094         List<Uint64> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
2095
2096         if (targetDpns.isEmpty()) {
2097             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
2098             return;
2099         }
2100
2101         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2102         java.util.Optional<Uint32> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2103
2104         // delete from FIB
2105         //
2106         optNextHop.ifPresent(nextHop -> {
2107             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2108             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2109             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
2110                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2111
2112             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
2113             for (Uint64 dpId : targetDpns) {
2114                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2115                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
2116
2117                 mdsalManager.removeFlow(dpId, flow);
2118             }
2119         });
2120
2121         // delete from LFIB
2122         //
2123         optLabel.ifPresent(label -> {
2124             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2125
2126             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
2127                 for (Uint64 dpId : targetDpns) {
2128                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2129                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
2130                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY,
2131                             NwConstants.DEL_FLOW, tx);
2132                 }
2133             }), LOG, "Error removing flows");
2134         });
2135     }
2136
2137     private static boolean isPrefixAndNextHopPresentInLri(String prefix,
2138             List<String> nextHopAddressList, LabelRouteInfo lri) {
2139         return lri != null && Objects.equals(lri.getPrefix(), prefix)
2140                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2141     }
2142
2143     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Uint32 vpnId, VrfEntry vrfEntry, Uint64 dpnId) {
2144         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2145             return true;
2146         }
2147
2148         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2149         if (prefix != null) {
2150             Uint64 prefixDpnId = prefix.getDpnId();
2151             if (dpnId.equals(prefixDpnId)) {
2152                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2153                         vrfEntry, dpnId);
2154                 return false;
2155             }
2156         }
2157         return true;
2158     }
2159
2160     private static ReentrantLock lockFor(final VpnInstanceOpDataEntry vpnInstance) {
2161         // FIXME: use vpnInstance.key() instead?
2162         return JvmGlobalLocks.getLockForString(vpnInstance.getVpnInstanceName());
2163     }
2164
2165     private static ReentrantLock lockFor(LabelRouteInfoKey label) {
2166         return JvmGlobalLocks.getLockFor(label);
2167     }
2168 }