fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.math.BigInteger;
  23 import java.net.InetAddress;
  24 import java.net.UnknownHostException;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Collection;
  28 import java.util.Collections;
  29 import java.util.List;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.CopyOnWriteArrayList;
  32 import java.util.concurrent.ExecutionException;
  33 import javax.annotation.PostConstruct;
  34 import javax.inject.Inject;
  35 import javax.inject.Singleton;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  40 import org.opendaylight.genius.infra.Datastore.Configuration;
  41 import org.opendaylight.genius.infra.Datastore.Operational;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  44 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.TransactionAdapter;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.mdsalutil.ActionInfo;
  49 import org.opendaylight.genius.mdsalutil.FlowEntity;
  50 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  51 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  52 import org.opendaylight.genius.mdsalutil.MatchInfo;
  53 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  54 import org.opendaylight.genius.mdsalutil.NWUtil;
  55 import org.opendaylight.genius.mdsalutil.NwConstants;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  62 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  68 import org.opendaylight.genius.utils.ServiceIndex;
  69 import org.opendaylight.genius.utils.batching.SubTransaction;
  70 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  71 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  72 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  73 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  74 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  75 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  76 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  77 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  78 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  79 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  80 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 114 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 115 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119
 120 @Singleton
 121 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 122
 123     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 124     private static final String FLOWID_PREFIX = "L3.";
 125     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 126     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 127     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 128     private static final int LFIB_INTERVPN_PRIORITY = 15;
 129     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 130     private static final int MAX_RETRIES = 3;
 131     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 136     private final IMdsalApiManager mdsalManager;
 137     private final NexthopManager nextHopManager;
 138     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 139     private final BaseVrfEntryHandler baseVrfEntryHandler;
 140     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final IElanService elanManager;
 143     private final FibUtil fibUtil;
 144     private final InterVpnLinkCache interVpnLinkCache;
 145     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 146     private final UpgradeState upgradeState;
 147     private final DataTreeEventCallbackRegistrar eventCallbacks;
 148
 149     @Inject
 150     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 151                             final NexthopManager nexthopManager,
 152                             final IElanService elanManager,
 153                             final BaseVrfEntryHandler vrfEntryHandler,
 154                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 155                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 156                             final JobCoordinator jobCoordinator,
 157                             final FibUtil fibUtil,
 158                             final InterVpnLinkCache interVpnLinkCache,
 159                             final UpgradeState upgradeState,
 160                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 161         super(VrfEntry.class, VrfEntryListener.class);
 162         this.dataBroker = dataBroker;
 163         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 164         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 165         this.mdsalManager = mdsalApiManager;
 166         this.nextHopManager = nexthopManager;
 167         this.elanManager = elanManager;
 168         this.baseVrfEntryHandler = vrfEntryHandler;
 169         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 170         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 171         this.jobCoordinator = jobCoordinator;
 172         this.fibUtil = fibUtil;
 173         this.interVpnLinkCache = interVpnLinkCache;
 174         this.upgradeState = upgradeState;
 175         this.eventCallbacks = eventCallbacks;
 176     }
 177
 178     @Override
 179     @PostConstruct
 180     public void init() {
 181         LOG.info("{} init", getClass().getSimpleName());
 182         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 183     }
 184
 185     @Override
 186     @SuppressWarnings("checkstyle:IllegalCatch")
 187     public void close() {
 188         closeables.forEach(c -> {
 189             try {
 190                 c.close();
 191             } catch (Exception e) {
 192                 LOG.warn("Error closing {}", c, e);
 193             }
 194         });
 195     }
 196
 197     @Override
 198     protected VrfEntryListener getDataTreeChangeListener() {
 199         return VrfEntryListener.this;
 200     }
 201
 202     @Override
 203     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 204         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 205     }
 206
 207     @Override
 208     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 209         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 210         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 211         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 212                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 213         addFibEntries(identifier, vrfEntry, rd);
 214         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 215                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 216     }
 217
 218     //This method is temporary. Eventually Factory design pattern will be used to get
 219     // right VrfEntryhandle and invoke its methods.
 220     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 221         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 222             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 223             return;
 224         }
 225         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 226             LOG.info("EVPN flows need to be programmed.");
 227             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 228                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 229             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 230             closeables.add(evpnVrfEntryHandler);
 231             return;
 232         }
 233         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 234         if (routerInt != null) {
 235             // ping responder for router interfaces
 236             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 237             return;
 238         }
 239         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 240             createFibEntries(identifier, vrfEntry);
 241             return;
 242         }
 243     }
 244
 245     @Override
 246     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 247         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 248         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 249         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 250                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 251         removeFibEntries(identifier, vrfEntry, rd);
 252         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 253             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 254     }
 255
 256     //This method is temporary. Eventually Factory design pattern will be used to get
 257     // right VrfEntryhandle and invoke its methods.
 258     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 259         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 260             LOG.info("EVPN flows to be deleted");
 261             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 262                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 263             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 264             closeables.add(evpnVrfEntryHandler);
 265             return;
 266         }
 267         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 268         if (routerInt != null) {
 269             // ping responder for router interfaces
 270             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 271             return;
 272         }
 273         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 274             deleteFibEntries(identifier, vrfEntry);
 275             return;
 276         }
 277         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 278             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 279             return;
 280         }
 281     }
 282
 283     @Override
 284     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 285     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 286     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 287     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 288         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 289         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 290         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 291                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 292         // Handle BGP Routes first
 293         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 294             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 295             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 296                     rd, update.getDestPrefix(), update.getRoutePaths());
 297             return;
 298         }
 299
 300         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 301             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 302             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 303             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 304
 305             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 306             //updated vrf entry route path is null. This can happen during tunnel events.
 307             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 308             List<String> usedRds = new ArrayList<>();
 309             if (optVpnInstance.isPresent()) {
 310                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 311                         update.getDestPrefix());
 312             }
 313             // If original VRF Entry had nexthop null , but update VRF Entry
 314             // has nexthop , route needs to be created on remote Dpns
 315             if (originalRoutePath == null || originalRoutePath.isEmpty()
 316                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 317                 // TODO(vivek): Though ugly, Not handling this code now, as each
 318                 // tep add event will invoke flow addition
 319                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 320                         + " This event is IGNORED here.", update.getDestPrefix());
 321                 return;
 322             }
 323
 324             // If original VRF Entry had valid nexthop , but update VRF Entry
 325             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 326             if (updateRoutePath == null || updateRoutePath.isEmpty()
 327                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 328                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 329                         + "This event is IGNORED here.", update.getDestPrefix());
 330                 return;
 331             }
 332             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 333             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 334             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 335             List<ListenableFuture<Void>> futures = new ArrayList<>();
 336             ListenableFuture<Void> configFuture =
 337                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 338                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 339                         nextHopsRemoved.parallelStream()
 340                             .forEach(nextHopRemoved -> {
 341                                 try {
 342                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 343                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 344                                 } catch (ExecutionException | InterruptedException e) {
 345                                     throw new RuntimeException(e);
 346                                 }
 347                             }))));
 348             futures.add(configFuture);
 349             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 350                 @Override
 351                 public void onSuccess(Void result) {
 352                     createFibEntries(identifier, update);
 353                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 354                             rd, update.getDestPrefix(), update.getRoutePaths());
 355                 }
 356
 357                 @Override
 358                 public void onFailure(Throwable throwable) {
 359                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 360                             update, throwable);
 361                 }
 362             }, MoreExecutors.directExecutor());
 363             return;
 364         }
 365
 366         //Handle all other routes only on a cluster reboot
 367         if (original.equals(update)) {
 368             //Reboot use-case
 369             createFibEntries(identifier, update);
 370             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 371                     rd, update.getDestPrefix(), update.getRoutePaths());
 372             return;
 373         }
 374
 375         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 376                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 377     }
 378
 379     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 380         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 381         List<SubTransaction> txnObjects =  new ArrayList<>();
 382         final VpnInstanceOpDataEntry vpnInstance =
 383                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 384         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 385         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 386                 + " has null vpnId!");
 387         final Collection<VpnToDpnList> vpnToDpnList;
 388         if (vrfEntry.getParentVpnRd() != null
 389                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 390             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 391             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 392             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 393                 vpnInstance.getVpnToDpnList();
 394             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 395                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 396         } else {
 397             vpnToDpnList = vpnInstance.getVpnToDpnList();
 398         }
 399         final Long vpnId = vpnInstance.getVpnId();
 400         final String rd = vrfTableKey.getRouteDistinguisher();
 401         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 402         if (subnetRoute != null) {
 403             final long elanTag = subnetRoute.getElantag();
 404             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 405                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 406             if (vpnToDpnList != null) {
 407                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 408                     () -> Collections.singletonList(
 409                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 410                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 411                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 412                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 413                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 414                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 415                                 }
 416                             }
 417                         })));
 418             }
 419             return;
 420         }
 421         // Get etherType value based on the IpPrefix address family type
 422         int etherType;
 423         try {
 424             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 425         } catch (IllegalArgumentException ex) {
 426             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 427             return;
 428         }
 429
 430         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 431         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 432             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 433                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 434                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 435                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 436                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 437                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 438                                     try {
 439                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 440                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 441                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 442                                                     TransactionAdapter.toWriteTransaction(tx),
 443                                                     txnObjects);
 444                                         } else {
 445                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 446                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 447                                         }
 448                                     } catch (NullPointerException e) {
 449                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 450                                                 vrfEntry.getDestPrefix(), e);
 451                                     }
 452                                 }
 453                             }
 454                         }
 455                     }
 456                 })), MAX_RETRIES);
 457         }
 458
 459         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 460         if (optVpnUuid.isPresent()) {
 461             String vpnUuid = optVpnUuid.get();
 462             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 463             if (interVpnLink != null) {
 464                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 465                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 466                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 467                         // This is an static route that points to the other endpoint of an InterVpnLink
 468                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 469                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 470                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 471                     }
 472                 });
 473             }
 474         }
 475     }
 476
 477     void refreshFibTables(String rd, String prefix) {
 478         InstanceIdentifier<VrfEntry> vrfEntryId =
 479                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 480                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 481         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 482         if (vrfEntry.isPresent()) {
 483             createFibEntries(vrfEntryId, vrfEntry.get());
 484         }
 485     }
 486
 487     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 488         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 489         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 490         prefixBuilder.setDpnId(lri.getDpnId());
 491         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 492         prefixBuilder.setIpAddress(lri.getPrefix());
 493         // Increment the refCount here
 494         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 495             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 496         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 497         if (!isPresentInList) {
 498             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 499             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 500             vpnInstanceNames.add(vpnInstanceName);
 501             builder.setVpnInstanceList(vpnInstanceNames);
 502             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 503         } else {
 504             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 505         }
 506         return prefixBuilder.build();
 507     }
 508
 509     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 510             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 511         if (tx == null) {
 512             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 513                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 514                 "Error installing subnet route in FIB");
 515             return;
 516         }
 517         int etherType;
 518         try {
 519             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 520         } catch (IllegalArgumentException ex) {
 521             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 522             return;
 523         }
 524         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 525             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 526             synchronized (label.toString().intern()) {
 527                 LabelRouteInfo lri = getLabelRouteInfo(label);
 528                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 529
 530                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 531                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 532                                 fibUtil.getVpnInstanceOpData(rd);
 533                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 534                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 535                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 536                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 537                             }
 538                         }
 539                     }
 540                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 541                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 542                 }
 543             }
 544         });
 545         final List<InstructionInfo> instructions = new ArrayList<>();
 546         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 547             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 548         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 549         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 550         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 551                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 552         if (vrfEntry.getRoutePaths() != null) {
 553             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 554                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 555                     List<ActionInfo> actionsInfos = new ArrayList<>();
 556                     // reinitialize instructions list for LFIB Table
 557                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 558                     actionsInfos.add(new ActionPopMpls(etherType));
 559                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 560                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 561                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 562                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 563
 564                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 565                             NwConstants.ADD_FLOW, tx);
 566                 }
 567             }
 568         }
 569     }
 570
 571     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 572             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 573         List<MatchInfo> matches = new ArrayList<>();
 574
 575         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 576                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 577         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 578         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 579         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 580
 581         InetAddress destPrefix;
 582         try {
 583             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 584         } catch (UnknownHostException e) {
 585             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 586                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 587             return;
 588         }
 589
 590         // Match on VpnId and SubnetBroadCast IP address
 591         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 592         matches.add(MatchEthernetType.IPV4);
 593
 594         if (prefixLength != 0) {
 595             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 596         }
 597
 598         //Action is to drop the packet
 599         List<InstructionInfo> dropInstructions = new ArrayList<>();
 600         List<ActionInfo> actionsInfos = new ArrayList<>();
 601         actionsInfos.add(new ActionDrop());
 602         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 603
 604         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 605         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 606         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 607                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 608
 609         Flow flow = flowEntity.getFlowBuilder().build();
 610         String flowId = flowEntity.getFlowId();
 611         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 612         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 613
 614         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 615                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 616                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 617
 618         if (addOrRemove == NwConstants.ADD_FLOW) {
 619             tx.put(flowInstanceId,flow, true);
 620         } else {
 621             tx.delete(flowInstanceId);
 622         }
 623     }
 624
 625     /*
 626      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 627      * LportDispatcher table (via table 80)
 628      */
 629     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 630                                             final VrfEntry vrfEntry, int etherType) {
 631         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 632         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 633         // packet is commuted from Vpn2 to Vpn1.
 634         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 635         if (!interVpnLink.isActive()) {
 636             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 637                      interVpnLinkName, vrfEntry.getDestPrefix());
 638             return;
 639         }
 640
 641         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 642         if (!optLportTag.isPresent()) {
 643             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 644             return;
 645         }
 646
 647         Long lportTag = optLportTag.get();
 648         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 649         if (label == null) {
 650             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 651                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 652             return;
 653         }
 654         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 655         List<InstructionInfo> instructions = Arrays.asList(
 656             new InstructionApplyActions(actionsInfos),
 657             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 658                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 659                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 660                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 661             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 662         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 663         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 664
 665         for (BigInteger dpId : targetDpns) {
 666             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 667                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 668
 669             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 670                                /*writeTx*/null);
 671         }
 672     }
 673
 674
 675     /*
 676      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 677      */
 678     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 679                                                final VrfEntry vrfEntry, long vpnTag) {
 680         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 681         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 682             && vrfEntry.getRoutePaths().size() == 1);
 683         String destination = vrfEntry.getDestPrefix();
 684         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 685         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 686
 687         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 688         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 689         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 690         if (interVpnLink.getState().or(State.Error) != State.Active) {
 691             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 692                 destination, nextHop, interVpnLinkName);
 693             return;
 694         }
 695
 696         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 697         if (!optOtherEndpointLportTag.isPresent()) {
 698             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 699                 vpnUuid, interVpnLinkName);
 700             return;
 701         }
 702
 703         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 704         if (targetDpns.isEmpty()) {
 705             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 706                 vpnUuid, interVpnLinkName);
 707             return;
 708         }
 709
 710         String[] values = destination.split("/");
 711         String destPrefixIpAddress = values[0];
 712         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 713
 714         List<MatchInfo> matches = new ArrayList<>();
 715         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 716         matches.add(MatchEthernetType.IPV4);
 717
 718         if (prefixLength != 0) {
 719             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 720         }
 721
 722         List<Instruction> instructions =
 723             Arrays.asList(new InstructionWriteMetadata(
 724                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 725                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 726                             .L3VPN_SERVICE_INDEX)),
 727                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 728                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 729
 730         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 731         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 732         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 733             COOKIE_VM_FIB_TABLE, matches, instructions);
 734
 735         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 736             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 737
 738         for (BigInteger dpId : targetDpns) {
 739
 740             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 741                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 742                 dpId, interVpnLink.getInterVpnLinkName());
 743
 744             mdsalManager.installFlow(dpId, flowEntity);
 745         }
 746     }
 747
 748     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 749         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 750         String localNextHopIP = vrfEntry.getDestPrefix();
 751         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 752         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 753         if (localNextHopInfo == null) {
 754             boolean localNextHopSeen = false;
 755             List<Routes> vpnExtraRoutes = null;
 756             String rdPrefixKey = localNextHopIP + FibConstants.SEPARATOR + rd;
 757             //Synchronized to prevent missing bucket action due to race condition between refreshFib and
 758             // add/updateFib threads on missing nexthop in VpnToExtraroutes
 759             synchronized (rdPrefixKey.intern()) {
 760                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 761                 vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 762                         vpnName, usedRds, localNextHopIP);
 763                 if (LOG.isDebugEnabled()) {
 764                     LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 765                             vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 766                 }
 767
 768                 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 769                 for (Routes vpnExtraRoute : vpnExtraRoutes) {
 770                     String ipPrefix;
 771                     if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 772                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 773                     } else {
 774                         ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 775                     }
 776                     Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 777                             ipPrefix);
 778                     if (localNextHopInfoLocal != null) {
 779                         localNextHopSeen = true;
 780                         BigInteger dpnId =
 781                                 checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 782                                         vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 783                         returnLocalDpnId.add(dpnId);
 784                     }
 785                 }
 786             }
 787             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 788                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 789                 if (optionalLabel.isPresent()) {
 790                     Long label = optionalLabel.get();
 791                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 792                     synchronized (label.toString().intern()) {
 793                         LabelRouteInfo lri = getLabelRouteInfo(label);
 794                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 795                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 796                                     fibUtil.getVpnInstanceOpData(rd);
 797                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 798                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 799                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 800                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 801                                     localNextHopIP = lri.getPrefix();
 802                                 } else {
 803                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 804                                     localNextHopIP = lri.getPrefix();
 805                                 }
 806                             }
 807                             if (localNextHopInfo != null) {
 808                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 809                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 810                                 if (vpnExtraRoutes.isEmpty()) {
 811                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 812                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 813                                     returnLocalDpnId.add(dpnId);
 814                                 } else {
 815                                     for (Routes extraRoutes : vpnExtraRoutes) {
 816                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 817                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 818                                         returnLocalDpnId.add(dpnId);
 819                                     }
 820                                 }
 821                             }
 822                         }
 823                     }
 824                 }
 825             }
 826             if (returnLocalDpnId.isEmpty()) {
 827                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 828             }
 829         } else {
 830             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 831                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 832             if (dpnId != null) {
 833                 returnLocalDpnId.add(dpnId);
 834             }
 835         }
 836         return returnLocalDpnId;
 837     }
 838
 839     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 840                                                 final Long vpnId, final String rd,
 841                                                 final VrfEntry vrfEntry,
 842                                                 Routes routes, List<Routes> vpnExtraRoutes,
 843                                                 int etherType) {
 844         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 845         if (localNextHopInfo != null) {
 846             long groupId;
 847             long localGroupId;
 848             final BigInteger dpnId = localNextHopInfo.getDpnId();
 849             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 850                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 851                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 852                 return dpnId;
 853             }
 854             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 855                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 856                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 857                 return dpnId;
 858             }
 859             if (!isVpnPresentInDpn(rd, dpnId)) {
 860                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 861                         vpnId, rd, dpnId.toString());
 862                 return BigInteger.ZERO;
 863             }
 864             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 865             String prefix = vrfEntry.getDestPrefix();
 866             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 867             //The loadbalancing group is created only if the extra route has multiple nexthops
 868             //to avoid loadbalancing the discovered routes
 869             if (vpnExtraRoutes != null && routes != null) {
 870                 if (vpnExtraRoutes.size() > 1) {
 871                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 872                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 873                 } else {
 874                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 875                     localGroupId = groupId;
 876                 }
 877             } else {
 878                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 879                         gwMacAddress);
 880                 localGroupId = groupId;
 881             }
 882             if (groupId == FibConstants.INVALID_GROUP_ID) {
 883                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 884                         prefix, rd, interfaceName, dpnId.toString());
 885                 return BigInteger.ZERO;
 886             }
 887             final List<InstructionInfo> instructions = Collections.singletonList(
 888                     new InstructionApplyActions(
 889                             Collections.singletonList(new ActionGroup(groupId))));
 890             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 891                     new InstructionApplyActions(
 892                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 893             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 894             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 895             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 896             jobCoordinator.enqueueJob(jobKey,
 897                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 898                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 899                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 900                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 901                         optLabel.ifPresent(label -> {
 902                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 903                                 LOG.debug(
 904                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 905                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 906                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 907                                         nextHopAddressList);
 908                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 909                                         NwConstants.ADD_FLOW, tx);
 910                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 911                             } else {
 912                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 913                                                 + "route. LFib and Terminating table entries will not be created.",
 914                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 915                             }
 916                         });
 917                     }
 918                 })));
 919             return dpnId;
 920         }
 921         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 922                 vpnName);
 923         return BigInteger.ZERO;
 924     }
 925
 926     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 927         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 928         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 929         if (dpnInVpn.isPresent()) {
 930             return true;
 931         }
 932         return false;
 933     }
 934
 935     private LabelRouteInfo getLabelRouteInfo(Long label) {
 936         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 937             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 938         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 939         if (opResult.isPresent()) {
 940             return opResult.get();
 941         }
 942         return null;
 943     }
 944
 945     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 946             TypedWriteTransaction<Operational> tx) {
 947         if (lri == null) {
 948             return true;
 949         }
 950
 951         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 952         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 953             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 954
 955         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 956             ? lri.getVpnInstanceList() : new ArrayList<>();
 957         if (vpnInstancesList.contains(vpnInstanceName)) {
 958             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 959             vpnInstancesList.remove(vpnInstanceName);
 960         }
 961         if (vpnInstancesList.isEmpty()) {
 962             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 963             if (tx != null) {
 964                 tx.delete(lriId);
 965             } else {
 966                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 967             }
 968             return true;
 969         } else {
 970             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 971             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 972             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 973         }
 974         return false;
 975     }
 976
 977     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 978                                       TypedWriteTransaction<Configuration> tx) {
 979         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 980
 981         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 982
 983         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 984             dpId, label, groupId);
 985     }
 986
 987     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 988                                                 TypedWriteTransaction<Configuration> tx) {
 989         List<MatchInfo> mkMatches = new ArrayList<>();
 990
 991         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 992             destDpId, label, actionsInfos);
 993
 994         // Matching metadata
 995         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 996         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 997
 998         List<InstructionInfo> mkInstructions = new ArrayList<>();
 999         mkInstructions.add(new InstructionApplyActions(actionsInfos));
1000
1001         FlowEntity terminatingServiceTableFlowEntity =
1002             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1003             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1004                 String.format("%s:%d", "TST Flow Entry ", label),
1005             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1006
1007         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1008
1009         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1010
1011         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1012         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1013             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1014             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1015             .child(Flow.class, flowKey).build();
1016         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1017     }
1018
1019     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1020         FlowEntity flowEntity;
1021         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1022         List<MatchInfo> mkMatches = new ArrayList<>();
1023         // Matching metadata
1024         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1025         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1026             NwConstants.INTERNAL_TUNNEL_TABLE,
1027             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1028             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1029             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1030         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1031         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1032         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1033             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1034             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1035
1036         tx.delete(flowInstanceId);
1037         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1038     }
1039
1040     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1041         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1042         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1043         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1044         boolean shouldUpdateNonEcmpLocalNextHop = true;
1045         if (localNextHopInfo == null) {
1046             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1047             if (usedRds.size() > 1) {
1048                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1049                         vrfEntry.getDestPrefix(), vpnName, rd);
1050                 return returnLocalDpnId;
1051             }
1052             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1053             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1054             //in the vpn
1055             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1056                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1057             if (extraRouteOptional.isPresent()) {
1058                 Routes extraRoute = extraRouteOptional.get();
1059                 String ipPrefix;
1060                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1061                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1062                 } else {
1063                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1064                 }
1065                 if (extraRoute.getNexthopIpList().size() > 1) {
1066                     shouldUpdateNonEcmpLocalNextHop = false;
1067                 }
1068                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1069                 if (localNextHopInfo != null) {
1070                     String localNextHopIP = localNextHopInfo.getIpAddress();
1071                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1072                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1073                     if (!dpnId.equals(BigInteger.ZERO)) {
1074                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1075                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1076                         returnLocalDpnId.add(dpnId);
1077                     }
1078                 } else {
1079                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1080                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1081                 }
1082             }
1083
1084             if (localNextHopInfo == null) {
1085                 /* Imported VRF entry */
1086                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1087                 if (optionalLabel.isPresent()) {
1088                     Long label = optionalLabel.get();
1089                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1090                     LabelRouteInfo lri = getLabelRouteInfo(label);
1091                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1092                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1093                         prefixBuilder.setDpnId(lri.getDpnId());
1094                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1095                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1096                         if (!dpnId.equals(BigInteger.ZERO)) {
1097                             returnLocalDpnId.add(dpnId);
1098                         }
1099                     }
1100                 }
1101             }
1102
1103         } else {
1104             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1105             String localNextHopIP = localNextHopInfo.getIpAddress();
1106             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1107                     shouldUpdateNonEcmpLocalNextHop);
1108             if (!dpnId.equals(BigInteger.ZERO)) {
1109                 returnLocalDpnId.add(dpnId);
1110             }
1111         }
1112
1113         return returnLocalDpnId;
1114     }
1115
1116     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1117             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1118             boolean shouldUpdateNonEcmpLocalNextHop) {
1119         if (localNextHopInfo != null) {
1120             final BigInteger dpnId = localNextHopInfo.getDpnId();
1121             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1122                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1123                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1124                 return dpnId;
1125             }
1126             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1127                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1128                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1129                 return dpnId;
1130             }
1131
1132             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1133                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1134                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1135                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1136                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1137                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1138                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1139                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1140                                         NwConstants.DEL_FLOW, tx);
1141                                 removeTunnelTableEntry(dpnId, label, tx);
1142                             });
1143                         }
1144                     }
1145                 })));
1146             //TODO: verify below adjacency call need to be optimized (?)
1147             //In case of the removal of the extra route, the loadbalancing group is updated
1148             if (shouldUpdateNonEcmpLocalNextHop) {
1149                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1150             }
1151             return dpnId;
1152         }
1153         return BigInteger.ZERO;
1154     }
1155
1156     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1157             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1158         if (tx == null) {
1159             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1160                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1161                 "Error creating remote FIB entry");
1162             return;
1163         }
1164
1165         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1166         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1167                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1168
1169         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1170         if (adjacencyResults.isEmpty()) {
1171             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1172                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1173             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1174             return;
1175         }
1176
1177         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1178         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1179                 vpnName, usedRds, vrfEntry.getDestPrefix());
1180         // create loadbalancing groups for extra routes only when the extra route is present behind
1181         // multiple VMs
1182         if (!vpnExtraRoutes.isEmpty()) {
1183             List<InstructionInfo> instructions = new ArrayList<>();
1184             // Obtain the local routes for this particular dpn.
1185             java.util.Optional<Routes> routes = vpnExtraRoutes
1186                     .stream()
1187                     .filter(route -> {
1188                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1189                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1190                         if (prefixToInterface == null) {
1191                             return false;
1192                         }
1193                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1194                     }).findFirst();
1195             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1196                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1197             if (groupId == FibConstants.INVALID_GROUP_ID) {
1198                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1199                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1200                 return;
1201             }
1202             List<ActionInfo> actionInfos =
1203                     Collections.singletonList(new ActionGroup(groupId));
1204             instructions.add(new InstructionApplyActions(actionInfos));
1205             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1206                     NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1207         } else {
1208             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry,
1209                 TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null);
1210         }
1211
1212         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1213     }
1214
1215     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1216     /* Get interface info from prefix to interface mapping;
1217         Use the interface info to get the corresponding vpn interface op DS entry,
1218         remove the adjacency corresponding to this fib entry.
1219         If adjacency removed is the last adjacency, clean up the following:
1220          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1221          - prefix to interface entry
1222          - vpn interface op DS
1223      */
1224         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1225         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1226         if (prefixInfo == null) {
1227             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1228             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1229             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1230             if (extraRoute != null) {
1231                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1232                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1233                     if (nextHopIp != null) {
1234                         String ipPrefix;
1235                         if (isIpv4Address(nextHopIp)) {
1236                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1237                         } else {
1238                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1239                         }
1240                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1241                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1242                     }
1243                 }
1244             }
1245             if (prefixInfo == null) {
1246                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1247                 if (optionalLabel.isPresent()) {
1248                     Long label = optionalLabel.get();
1249                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1250                     LabelRouteInfo lri = getLabelRouteInfo(label);
1251                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1252                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1253                         prefixBuilder.setDpnId(lri.getDpnId());
1254                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1255                         prefixBuilder.setIpAddress(lri.getPrefix());
1256                         prefixInfo = prefixBuilder.build();
1257                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1258                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1259                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1260                     }
1261                 }
1262             }
1263         } else {
1264             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1265         }
1266     }
1267
1268     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1269                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1270
1271         if (prefixInfo == null) {
1272             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1273                     vrfEntry.getDestPrefix(), vpnId, rd);
1274             return; //Don't have any info for this prefix (shouldn't happen); need to return
1275         }
1276
1277         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1278             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1279                     vrfEntry.getDestPrefix(), vpnId, rd);
1280             return;
1281         }
1282
1283         String ifName = prefixInfo.getVpnInterfaceName();
1284         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1285             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1286     }
1287
1288     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1289         Prefixes prefixInfo;
1290         Long vpnId;
1291         String rd;
1292         VrfEntry vrfEntry;
1293         Routes extraRoute;
1294
1295         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1296                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1297             this.prefixInfo = prefixInfo;
1298             this.vpnId = vpnId;
1299             this.rd = rd;
1300             this.vrfEntry = vrfEntry;
1301             this.extraRoute = extraRoute;
1302         }
1303
1304         @Override
1305         public List<ListenableFuture<Void>> call() {
1306             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1307             // to call the respective helpers.
1308             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1309                 //First Cleanup LabelRouteInfo
1310                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1311                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1312                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1313                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1314                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1315                         synchronized (label.toString().intern()) {
1316                             LabelRouteInfo lri = getLabelRouteInfo(label);
1317                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1318                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1319                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1320                                         fibUtil.getVpnInstanceOpData(rd);
1321                                 String vpnInstanceName = "";
1322                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1323                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1324                                 }
1325                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1326                                 if (lriRemoved) {
1327                                     String parentRd = lri.getParentVpnRd();
1328                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1329                                             parentRd, vrfEntry.getDestPrefix()));
1330                                 }
1331                             } else {
1332                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1333                                         rd, vrfEntry.getDestPrefix()));
1334                             }
1335                         }
1336                     });
1337                 }
1338                 String ifName = prefixInfo.getVpnInterfaceName();
1339                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1340                 String vpnName = null;
1341
1342                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1343                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1344                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1345                     if (vpnNameOpt.isPresent()) {
1346                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1347                     }
1348                 }
1349                 if (optVpnName.isPresent()) {
1350                     vpnName = optVpnName.get();
1351                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1352                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1353                     if (opVpnInterface.isPresent()) {
1354                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1355                         if (vpnId != associatedVpnId) {
1356                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1357                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1358                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1359                             return;
1360                         } else {
1361                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1362                                     vrfEntry.getDestPrefix(), associatedVpnId);
1363                         }
1364                     }
1365                 }
1366                 if (extraRoute != null) {
1367                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1368                     //Only one used Rd present in case of removal event
1369                     String usedRd = usedRds.get(0);
1370                     if (optVpnName.isPresent()) {
1371                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1372                                         vrfEntry.getDestPrefix()));
1373                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1374                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1375                     }
1376                 }
1377                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1378             }));
1379         }
1380     }
1381
1382     /**
1383      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1384      * Remove Adjacency from VPNInterfaceOpData.
1385      * if Adjacency != primary.
1386      * if Adjacency == primary , then mark it for deletion.
1387      * Remove entire VPNinterfaceOpData Entry.
1388      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1389      * @param vrfEntry - VrfEntry removed
1390      * @param ifName - Interface name from VRFentry
1391      * @param vpnName - VPN name of corresponding VRF
1392      * @param tx - ReadWrite Tx
1393      */
1394     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1395                                                           TypedReadWriteTransaction<Operational> tx)
1396             throws ExecutionException, InterruptedException {
1397         InstanceIdentifier<Adjacency> adjacencyIid =
1398                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1399         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1400         if (adjacencyOptional.isPresent()) {
1401             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1402                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1403             } else {
1404                 tx.merge(adjacencyIid,
1405                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1406             }
1407         }
1408
1409         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1410
1411         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1412             return;
1413         }
1414
1415         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1416                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1417                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1418                         && adjacency.isMarkedForDeletion() != null
1419                         && adjacency.isMarkedForDeletion()
1420         )) {
1421             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1422             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1423         }
1424     }
1425
1426     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1427         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1428         final String rd = vrfTableKey.getRouteDistinguisher();
1429         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1430         if (vpnInstance == null) {
1431             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1432             return;
1433         }
1434         final Collection<VpnToDpnList> vpnToDpnList;
1435         if (vrfEntry.getParentVpnRd() != null
1436                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1437             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1438             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1439             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1440                     vpnInstance.getVpnToDpnList();
1441             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1442                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1443         } else {
1444             vpnToDpnList = vpnInstance.getVpnToDpnList();
1445         }
1446
1447         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1448         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1449         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1450         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1451         if (subnetRoute != null) {
1452             long elanTag = subnetRoute.getElantag();
1453             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1454                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1455             if (vpnToDpnList != null) {
1456                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1457                     () -> Collections.singletonList(
1458                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1459                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1460
1461                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1462                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1463                                     TransactionAdapter.toWriteTransaction(tx), null);
1464                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1465                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1466                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1467                                 }
1468
1469                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1470                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1471                             }
1472                         })));
1473             }
1474             optionalLabel.ifPresent(label -> {
1475                 synchronized (label.toString().intern()) {
1476                     LabelRouteInfo lri = getLabelRouteInfo(label);
1477                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1478                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1479                                 fibUtil.getVpnInstanceOpData(rd);
1480                         String vpnInstanceName = "";
1481                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1482                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1483                         }
1484                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1485                         if (lriRemoved) {
1486                             String parentRd = lri.getParentVpnRd();
1487                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1488                                     parentRd, vrfEntry.getDestPrefix()));
1489                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1490                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1491                         }
1492                     } else {
1493                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1494                                 rd, vrfEntry.getDestPrefix()));
1495                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1496                                 label, rd, vrfEntry.getDestPrefix());
1497                     }
1498                 }
1499             });
1500             return;
1501         }
1502
1503         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1504             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1505         if (vpnToDpnList != null) {
1506             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1507                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1508             String jobKey;
1509             Optional<Routes> extraRouteOptional;
1510             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1511             if (usedRds != null && !usedRds.isEmpty()) {
1512                 if (usedRds.size() > 1) {
1513                     LOG.error("The extra route prefix is still present in some DPNs");
1514                     return ;
1515                 } else {
1516                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1517                     //is not present in any other DPN
1518                     extraRouteOptional = VpnExtraRouteHelper
1519                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1520                 }
1521             } else {
1522                 extraRouteOptional = Optional.absent();
1523             }
1524
1525             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1526                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1527                     if (localDpnIdList.size() <= 0) {
1528                         for (VpnToDpnList curDpn : vpnToDpnList) {
1529                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1530                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1531                                 TransactionAdapter.toWriteTransaction(tx));
1532                         }
1533                     } else {
1534                         for (BigInteger localDpnId : localDpnIdList) {
1535                             for (VpnToDpnList curDpn : vpnToDpnList) {
1536                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1537                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1538                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1539                                         TransactionAdapter.toWriteTransaction(tx));
1540                                 }
1541                             }
1542                         }
1543                     }
1544                     nextHopManager.removeNextHopPointer(nextHopManager
1545                             .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1546                     nextHopManager.removeNextHopPointer(nextHopManager
1547                             .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix()));
1548                 })), MAX_RETRIES);
1549         }
1550
1551         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1552         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1553         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1554
1555         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1556         // of the interVpnLink.
1557         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1558         if (optVpnUuid.isPresent()) {
1559             String vpnUuid = optVpnUuid.get();
1560             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1561                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1562                 if (optInterVpnLink.isPresent()) {
1563                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1564                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1565                         // This is route that points to the other endpoint of an InterVpnLink
1566                         // In that case, we should look for the FIB table pointing to
1567                         // LPortDispatcher table and remove it.
1568                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1569                     }
1570                 }
1571             });
1572         }
1573
1574     }
1575
1576     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1577                                     int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1578         if (tx == null) {
1579             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1580                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1581                 "Error making LFIB table entry");
1582             return;
1583         }
1584
1585         List<MatchInfo> matches = new ArrayList<>();
1586         matches.add(MatchEthernetType.MPLS_UNICAST);
1587         matches.add(new MatchMplsLabel(label));
1588
1589         // Install the flow entry in L3_LFIB_TABLE
1590         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1591
1592         FlowEntity flowEntity;
1593         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1594             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1595         Flow flow = flowEntity.getFlowBuilder().build();
1596         String flowId = flowEntity.getFlowId();
1597         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1598         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1599         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1600             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1601             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1602
1603         if (addOrRemove == NwConstants.ADD_FLOW) {
1604             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1605         } else {
1606             tx.delete(flowInstanceId);
1607         }
1608
1609         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1610             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1611     }
1612
1613     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1614                                     final FutureCallback<List<Void>> callback) {
1615         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1616         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1617             () -> {
1618                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1619                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1620                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1621                         LogicalDatastoreType.CONFIGURATION, id);
1622                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1623                 if (!vrfTable.isPresent()) {
1624                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1625                     if (callback != null) {
1626                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1627                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1628                     }
1629                     return futures;
1630                 }
1631                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1632                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1633                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1634                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1635                             if (subnetRoute != null) {
1636                                 long elanTag = subnetRoute.getElantag();
1637                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1638                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1639                                         tx);
1640                                 continue;
1641                             }
1642                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1643                             if (routerInt != null) {
1644                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1645                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1646                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1647                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1648                                         NwConstants.ADD_FLOW);
1649                                 continue;
1650                             }
1651                             //Handle local flow creation for imports
1652                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1653                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1654                                 if (optionalLabel.isPresent()) {
1655                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1656                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1657                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1658                                         if (lri.getDpnId().equals(dpnId)) {
1659                                             try {
1660                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1661                                                         vrfEntry.getDestPrefix());
1662                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1663                                             } catch (IllegalArgumentException ex) {
1664                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1665                                                         vrfEntry.getDestPrefix());
1666                                             }
1667                                             continue;
1668                                         }
1669                                     }
1670                                 }
1671                             }
1672                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1673                                     vrfEntry, dpnId);
1674                             if (shouldCreateRemoteFibEntry) {
1675                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1676                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1677                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1678                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1679                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1680                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1681                                 } else {
1682                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1683                                             vrfEntry, tx);
1684                                 }
1685                             }
1686                         }
1687                     }));
1688                     if (callback != null) {
1689                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1690                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1691                     }
1692                 }
1693                 return futures;
1694             });
1695     }
1696
1697     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1698                                             final String localNextHopIp, final String remoteNextHopIp) {
1699         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1700             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1701         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1702         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1703         List<SubTransaction> txnObjects =  new ArrayList<>();
1704         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1705         if (vrfTable.isPresent()) {
1706             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1707                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1708                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1709                         vrfTable.get().getVrfEntry().stream()
1710                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1711                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1712                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1713                     }
1714                 })));
1715         }
1716     }
1717
1718     public void manageRemoteRouteOnDPN(final boolean action,
1719                                        final BigInteger localDpnId,
1720                                        final long vpnId,
1721                                        final String rd,
1722                                        final String destPrefix,
1723                                        final String destTepIp,
1724                                        final long label) {
1725         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1726
1727         if (vpnInstance == null) {
1728             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1729             return;
1730         }
1731
1732         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1733             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1734                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1735                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1736                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1737                     if (vrfEntry == null) {
1738                         return;
1739                     }
1740                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1741                             action, localDpnId, vpnId, rd, destPrefix);
1742                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1743                     VrfEntry modVrfEntry;
1744                     if (routePathList == null || routePathList.isEmpty()) {
1745                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1746                                 Collections.singletonList(destTepIp),
1747                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1748                     } else {
1749                         modVrfEntry = vrfEntry;
1750                     }
1751
1752                     if (action) {
1753                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1754                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1755                                 modVrfEntry, tx);
1756                     } else {
1757                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1758                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1759                                 vrfEntry.getDestPrefix());
1760                         if (usedRds.size() > 1) {
1761                             LOG.debug("The extra route prefix is still present in some DPNs");
1762                             return;
1763                         }
1764                         //Is this fib route an extra route? If yes, get the nexthop which would be
1765                         //an adjacency in the vpn
1766                         Optional<Routes> extraRouteOptional = Optional.absent();
1767                         if (usedRds.size() != 0) {
1768                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1769                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1770                                     usedRds.get(0), vrfEntry.getDestPrefix());
1771                         }
1772                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1773                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1774                     }
1775                 }
1776             })));
1777     }
1778
1779     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1780                                  final FutureCallback<List<Void>> callback) {
1781         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1782         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1783             () -> {
1784                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1785                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1786                 List<SubTransaction> txnObjects =  new ArrayList<>();
1787                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1788                         LogicalDatastoreType.CONFIGURATION, id);
1789                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1790                 if (vrfTable.isPresent()) {
1791                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1792                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1793                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1794                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1795                                 /* Handle subnet routes here */
1796                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1797                                 if (subnetRoute != null && !fibUtil
1798                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1799                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1800                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1801                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1802                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1803                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1804                                     if (routePaths != null) {
1805                                         for (RoutePaths routePath : routePaths) {
1806                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1807                                                     DEFAULT_FIB_FLOW_PRIORITY,
1808                                                     NwConstants.DEL_FLOW, tx);
1809                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1810                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1811                                                     vrfEntry.getDestPrefix());
1812                                         }
1813                                     }
1814                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1815                                             NwConstants.DEL_FLOW, tx);
1816                                     continue;
1817                                 }
1818                                 // ping responder for router interfaces
1819                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1820                                 if (routerInt != null) {
1821                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1822                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1823                                             routerInt.getMacAddress());
1824                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1825                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1826                                             NwConstants.DEL_FLOW);
1827                                     continue;
1828                                 }
1829
1830                                 //Handle local flow deletion for imports
1831                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1832                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1833                                     if (optionalLabel.isPresent()) {
1834                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1835                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1836                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1837                                                 lri) && lri.getDpnId().equals(dpnId)) {
1838                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1839                                         }
1840                                     }
1841                                 }
1842
1843                                 // Passing null as we don't know the dpn
1844                                 // to which prefix is attached at this point
1845                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1846                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1847                                 Optional<Routes> extraRouteOptional;
1848                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1849                                 //an adjacency in the vpn
1850                                 if (usedRds != null && !usedRds.isEmpty()) {
1851                                     if (usedRds.size() > 1) {
1852                                         LOG.error("The extra route prefix is still present in some DPNs");
1853                                         return;
1854                                     } else {
1855                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1856                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1857
1858                                     }
1859                                 } else {
1860                                     extraRouteOptional = Optional.absent();
1861                                 }
1862                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1863                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1864                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1865                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1866                                 } else {
1867                                     if (subnetRoute == null || !fibUtil
1868                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1869                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1870                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1871                                             TransactionAdapter.toWriteTransaction(tx));
1872                                     }
1873                                 }
1874                             }
1875                         }));
1876                     }
1877                     if (callback != null) {
1878                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1879                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1880                     }
1881                 } else {
1882                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1883                 }
1884                 return futures;
1885             });
1886
1887     }
1888
1889     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1890                                            final String localNextHopIp, final String remoteNextHopIp) {
1891         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1892                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1893             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1894         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1895         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1896         List<SubTransaction> txnObjects =  new ArrayList<>();
1897         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1898         if (vrfTable.isPresent()) {
1899             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1900                 () -> {
1901                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1902                         return Collections.singletonList(
1903                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1904                                 tx -> vrfTable.get().getVrfEntry().stream()
1905                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1906                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1907                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1908                                         txnObjects))));
1909                     }
1910                 });
1911         }
1912     }
1913
1914     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1915         InstanceIdentifierBuilder<VrfTables> idBuilder =
1916             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1917         return idBuilder.build();
1918     }
1919
1920     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1921         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1922                 .FLOWID_SEPARATOR + nextHop;
1923     }
1924
1925     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1926         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1927                 + tableMiss + FLOWID_PREFIX;
1928     }
1929
1930     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1931         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1932             .child(VrfTables.class, new VrfTablesKey(rd))
1933             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1934         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1935         if (vrfEntry.isPresent()) {
1936             return vrfEntry.get();
1937         }
1938         return null;
1939     }
1940
1941     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1942                                              final String vpnName,
1943                                              final VrfEntry vrfEntry) {
1944         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1945
1946         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1947         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1948
1949         if (targetDpns.isEmpty()) {
1950             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1951             return;
1952         }
1953
1954         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1955         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1956
1957         // delete from FIB
1958         //
1959         optNextHop.ifPresent(nextHop -> {
1960             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1961             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1962             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1963                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1964
1965             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1966             for (BigInteger dpId : targetDpns) {
1967                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1968                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1969
1970                 mdsalManager.removeFlow(dpId, flow);
1971             }
1972         });
1973
1974         // delete from LFIB
1975         //
1976         optLabel.ifPresent(label -> {
1977             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1978
1979             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1980                 for (BigInteger dpId : targetDpns) {
1981                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1982                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1983                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1984                             tx);
1985                 }
1986             }), LOG, "Error removing flows");
1987         });
1988     }
1989
1990     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1991             List<String> nextHopAddressList, LabelRouteInfo lri) {
1992         return lri != null && lri.getPrefix().equals(prefix)
1993                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1994     }
1995
1996     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1997         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1998             return true;
1999         }
2000
2001         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
2002         if (prefix != null) {
2003             BigInteger prefixDpnId = prefix.getDpnId();
2004             if (dpnId.equals(prefixDpnId)) {
2005                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2006                         vrfEntry, dpnId);
2007                 return false;
2008             }
2009         }
2010         return true;
2011     }
2012 }