fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.FutureCallback;
  18 import com.google.common.util.concurrent.Futures;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import com.google.common.util.concurrent.MoreExecutors;
  21 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  22 import java.math.BigInteger;
  23 import java.net.InetAddress;
  24 import java.net.UnknownHostException;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Collection;
  28 import java.util.Collections;
  29 import java.util.List;
  30 import java.util.concurrent.Callable;
  31 import java.util.concurrent.CopyOnWriteArrayList;
  32 import java.util.concurrent.ExecutionException;
  33 import javax.annotation.PostConstruct;
  34 import javax.inject.Inject;
  35 import javax.inject.Singleton;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  40 import org.opendaylight.genius.infra.Datastore.Configuration;
  41 import org.opendaylight.genius.infra.Datastore.Operational;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  44 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.TransactionAdapter;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.mdsalutil.ActionInfo;
  49 import org.opendaylight.genius.mdsalutil.FlowEntity;
  50 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  51 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  52 import org.opendaylight.genius.mdsalutil.MatchInfo;
  53 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  54 import org.opendaylight.genius.mdsalutil.NWUtil;
  55 import org.opendaylight.genius.mdsalutil.NwConstants;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  62 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  68 import org.opendaylight.genius.utils.ServiceIndex;
  69 import org.opendaylight.genius.utils.batching.SubTransaction;
  70 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  71 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  72 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  73 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  74 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  75 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  76 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  77 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  78 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  79 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  80 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 114 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 115 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119
 120 @Singleton
 121 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 122
 123     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 124     private static final String FLOWID_PREFIX = "L3.";
 125     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 126     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 127     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 128     private static final int LFIB_INTERVPN_PRIORITY = 15;
 129     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 130     private static final int MAX_RETRIES = 3;
 131     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 136     private final IMdsalApiManager mdsalManager;
 137     private final NexthopManager nextHopManager;
 138     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 139     private final BaseVrfEntryHandler baseVrfEntryHandler;
 140     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final IElanService elanManager;
 143     private final FibUtil fibUtil;
 144     private final InterVpnLinkCache interVpnLinkCache;
 145     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 146     private final UpgradeState upgradeState;
 147     private final DataTreeEventCallbackRegistrar eventCallbacks;
 148
 149     @Inject
 150     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 151                             final NexthopManager nexthopManager,
 152                             final IElanService elanManager,
 153                             final BaseVrfEntryHandler vrfEntryHandler,
 154                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 155                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 156                             final JobCoordinator jobCoordinator,
 157                             final FibUtil fibUtil,
 158                             final InterVpnLinkCache interVpnLinkCache,
 159                             final UpgradeState upgradeState,
 160                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 161         super(VrfEntry.class, VrfEntryListener.class);
 162         this.dataBroker = dataBroker;
 163         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 164         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 165         this.mdsalManager = mdsalApiManager;
 166         this.nextHopManager = nexthopManager;
 167         this.elanManager = elanManager;
 168         this.baseVrfEntryHandler = vrfEntryHandler;
 169         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 170         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 171         this.jobCoordinator = jobCoordinator;
 172         this.fibUtil = fibUtil;
 173         this.interVpnLinkCache = interVpnLinkCache;
 174         this.upgradeState = upgradeState;
 175         this.eventCallbacks = eventCallbacks;
 176     }
 177
 178     @Override
 179     @PostConstruct
 180     public void init() {
 181         LOG.info("{} init", getClass().getSimpleName());
 182         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 183     }
 184
 185     @Override
 186     @SuppressWarnings("checkstyle:IllegalCatch")
 187     public void close() {
 188         closeables.forEach(c -> {
 189             try {
 190                 c.close();
 191             } catch (Exception e) {
 192                 LOG.warn("Error closing {}", c, e);
 193             }
 194         });
 195     }
 196
 197     @Override
 198     protected VrfEntryListener getDataTreeChangeListener() {
 199         return VrfEntryListener.this;
 200     }
 201
 202     @Override
 203     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 204         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 205     }
 206
 207     @Override
 208     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 209         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 210         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 211         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 212                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 213         addFibEntries(identifier, vrfEntry, rd);
 214         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 215                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 216     }
 217
 218     //This method is temporary. Eventually Factory design pattern will be used to get
 219     // right VrfEntryhandle and invoke its methods.
 220     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 221         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 222             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 223             return;
 224         }
 225         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 226             LOG.info("EVPN flows need to be programmed.");
 227             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 228                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 229             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 230             closeables.add(evpnVrfEntryHandler);
 231             return;
 232         }
 233         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 234         if (routerInt != null) {
 235             // ping responder for router interfaces
 236             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 237             return;
 238         }
 239         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 240             createFibEntries(identifier, vrfEntry);
 241             return;
 242         }
 243     }
 244
 245     @Override
 246     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 247         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 248         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 249         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 250                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 251         removeFibEntries(identifier, vrfEntry, rd);
 252         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 253             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 254     }
 255
 256     //This method is temporary. Eventually Factory design pattern will be used to get
 257     // right VrfEntryhandle and invoke its methods.
 258     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 259         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 260             LOG.info("EVPN flows to be deleted");
 261             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 262                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 263             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 264             closeables.add(evpnVrfEntryHandler);
 265             return;
 266         }
 267         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 268         if (routerInt != null) {
 269             // ping responder for router interfaces
 270             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 271             return;
 272         }
 273         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 274             deleteFibEntries(identifier, vrfEntry);
 275             return;
 276         }
 277         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 278             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 279             return;
 280         }
 281     }
 282
 283     @Override
 284     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 285     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 286     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 287     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 288         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 289         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 290         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 291                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 292         // Handle BGP Routes first
 293         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 294             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 295             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 296                     rd, update.getDestPrefix(), update.getRoutePaths());
 297             return;
 298         }
 299
 300         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 301             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 302             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 303             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 304
 305             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 306             //updated vrf entry route path is null. This can happen during tunnel events.
 307             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 308             List<String> usedRds = new ArrayList<>();
 309             if (optVpnInstance.isPresent()) {
 310                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 311                         update.getDestPrefix());
 312             }
 313             // If original VRF Entry had nexthop null , but update VRF Entry
 314             // has nexthop , route needs to be created on remote Dpns
 315             if (originalRoutePath == null || originalRoutePath.isEmpty()
 316                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 317                 // TODO(vivek): Though ugly, Not handling this code now, as each
 318                 // tep add event will invoke flow addition
 319                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 320                         + " This event is IGNORED here.", update.getDestPrefix());
 321                 return;
 322             }
 323
 324             // If original VRF Entry had valid nexthop , but update VRF Entry
 325             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 326             if (updateRoutePath == null || updateRoutePath.isEmpty()
 327                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 328                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 329                         + "This event is IGNORED here.", update.getDestPrefix());
 330                 return;
 331             }
 332             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 333             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 334             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 335             List<ListenableFuture<Void>> futures = new ArrayList<>();
 336             ListenableFuture<Void> configFuture =
 337                 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx ->
 338                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx ->
 339                         nextHopsRemoved.parallelStream()
 340                             .forEach(nextHopRemoved -> {
 341                                 try {
 342                                     fibUtil.updateUsedRdAndVpnToExtraRoute(
 343                                         configTx, operTx, nextHopRemoved, rd, update.getDestPrefix());
 344                                 } catch (ExecutionException | InterruptedException e) {
 345                                     throw new RuntimeException(e);
 346                                 }
 347                             }))));
 348             futures.add(configFuture);
 349             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 350                 @Override
 351                 public void onSuccess(Void result) {
 352                     createFibEntries(identifier, update);
 353                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 354                             rd, update.getDestPrefix(), update.getRoutePaths());
 355                 }
 356
 357                 @Override
 358                 public void onFailure(Throwable throwable) {
 359                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 360                             update, throwable);
 361                 }
 362             }, MoreExecutors.directExecutor());
 363             return;
 364         }
 365
 366         //Handle all other routes only on a cluster reboot
 367         if (original.equals(update)) {
 368             //Reboot use-case
 369             createFibEntries(identifier, update);
 370             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 371                     rd, update.getDestPrefix(), update.getRoutePaths());
 372             return;
 373         }
 374
 375         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 376                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 377     }
 378
 379     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 380         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 381         List<SubTransaction> txnObjects =  new ArrayList<>();
 382         final VpnInstanceOpDataEntry vpnInstance =
 383                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 384         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 385         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 386                 + " has null vpnId!");
 387         final Collection<VpnToDpnList> vpnToDpnList;
 388         if (vrfEntry.getParentVpnRd() != null
 389                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 390             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 391             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 392             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 393                 vpnInstance.getVpnToDpnList();
 394             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 395                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 396         } else {
 397             vpnToDpnList = vpnInstance.getVpnToDpnList();
 398         }
 399         final Long vpnId = vpnInstance.getVpnId();
 400         final String rd = vrfTableKey.getRouteDistinguisher();
 401         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 402         if (subnetRoute != null) {
 403             final long elanTag = subnetRoute.getElantag();
 404             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 405                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 406             if (vpnToDpnList != null) {
 407                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 408                     () -> Collections.singletonList(
 409                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 410                             for (final VpnToDpnList curDpn : vpnToDpnList) {
 411                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 412                                     installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 413                                     installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 414                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 415                                 }
 416                             }
 417                         })));
 418             }
 419             return;
 420         }
 421         // Get etherType value based on the IpPrefix address family type
 422         int etherType;
 423         try {
 424             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 425         } catch (IllegalArgumentException ex) {
 426             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 427             return;
 428         }
 429
 430         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 431         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 432             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 433                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 434                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 435                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 436                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 437                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 438                                     try {
 439                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 440                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 441                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry,
 442                                                     TransactionAdapter.toWriteTransaction(tx),
 443                                                     txnObjects);
 444                                         } else {
 445                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 446                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 447                                         }
 448                                     } catch (NullPointerException e) {
 449                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 450                                                 vrfEntry.getDestPrefix(), e);
 451                                     }
 452                                 }
 453                             }
 454                         }
 455                     }
 456                 })), MAX_RETRIES);
 457         }
 458
 459         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 460         if (optVpnUuid.isPresent()) {
 461             String vpnUuid = optVpnUuid.get();
 462             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 463             if (interVpnLink != null) {
 464                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 465                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 466                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 467                         // This is an static route that points to the other endpoint of an InterVpnLink
 468                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 469                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 470                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 471                     }
 472                 });
 473             }
 474         }
 475     }
 476
 477     void refreshFibTables(String rd, String prefix) {
 478         InstanceIdentifier<VrfEntry> vrfEntryId =
 479                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 480                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 481         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 482         if (vrfEntry.isPresent()) {
 483             createFibEntries(vrfEntryId, vrfEntry.get());
 484         }
 485     }
 486
 487     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 488         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 489         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 490         prefixBuilder.setDpnId(lri.getDpnId());
 491         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 492         prefixBuilder.setIpAddress(lri.getPrefix());
 493         // Increment the refCount here
 494         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 495             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 496         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 497         if (!isPresentInList) {
 498             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 499             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 500             vpnInstanceNames.add(vpnInstanceName);
 501             builder.setVpnInstanceList(vpnInstanceNames);
 502             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 503         } else {
 504             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 505         }
 506         return prefixBuilder.build();
 507     }
 508
 509     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 510             final long vpnId, final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
 511         if (tx == null) {
 512             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 513                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 514                 "Error installing subnet route in FIB");
 515             return;
 516         }
 517         int etherType;
 518         try {
 519             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 520         } catch (IllegalArgumentException ex) {
 521             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 522             return;
 523         }
 524         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 525             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 526             synchronized (label.toString().intern()) {
 527                 LabelRouteInfo lri = getLabelRouteInfo(label);
 528                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 529
 530                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 531                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 532                                 fibUtil.getVpnInstanceOpData(rd);
 533                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 534                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 535                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 536                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 537                             }
 538                         }
 539                     }
 540                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 541                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 542                 }
 543             }
 544         });
 545         final List<InstructionInfo> instructions = new ArrayList<>();
 546         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 547             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 548         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 549         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 550         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 551                 NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 552         if (vrfEntry.getRoutePaths() != null) {
 553             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 554                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 555                     List<ActionInfo> actionsInfos = new ArrayList<>();
 556                     // reinitialize instructions list for LFIB Table
 557                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 558                     actionsInfos.add(new ActionPopMpls(etherType));
 559                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 560                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 561                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 562                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 563
 564                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 565                             NwConstants.ADD_FLOW, tx);
 566                 }
 567             }
 568         }
 569     }
 570
 571     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 572             final VrfEntry vrfEntry, int addOrRemove, TypedWriteTransaction<Configuration> tx) {
 573         List<MatchInfo> matches = new ArrayList<>();
 574
 575         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 576                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 577         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 578         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 579         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 580
 581         InetAddress destPrefix;
 582         try {
 583             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 584         } catch (UnknownHostException e) {
 585             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 586                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 587             return;
 588         }
 589
 590         // Match on VpnId and SubnetBroadCast IP address
 591         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 592         matches.add(MatchEthernetType.IPV4);
 593
 594         if (prefixLength != 0) {
 595             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 596         }
 597
 598         //Action is to drop the packet
 599         List<InstructionInfo> dropInstructions = new ArrayList<>();
 600         List<ActionInfo> actionsInfos = new ArrayList<>();
 601         actionsInfos.add(new ActionDrop());
 602         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 603
 604         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 605         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 606         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 607                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 608
 609         Flow flow = flowEntity.getFlowBuilder().build();
 610         String flowId = flowEntity.getFlowId();
 611         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 612         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 613
 614         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 615                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 616                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 617
 618         if (addOrRemove == NwConstants.ADD_FLOW) {
 619             tx.put(flowInstanceId,flow, true);
 620         } else {
 621             tx.delete(flowInstanceId);
 622         }
 623     }
 624
 625     /*
 626      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 627      * LportDispatcher table (via table 80)
 628      */
 629     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 630                                             final VrfEntry vrfEntry, int etherType) {
 631         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 632         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 633         // packet is commuted from Vpn2 to Vpn1.
 634         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 635         if (!interVpnLink.isActive()) {
 636             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 637                      interVpnLinkName, vrfEntry.getDestPrefix());
 638             return;
 639         }
 640
 641         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 642         if (!optLportTag.isPresent()) {
 643             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 644             return;
 645         }
 646
 647         Long lportTag = optLportTag.get();
 648         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 649         if (label == null) {
 650             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 651                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 652             return;
 653         }
 654         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 655         List<InstructionInfo> instructions = Arrays.asList(
 656             new InstructionApplyActions(actionsInfos),
 657             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 658                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 659                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 660                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 661             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 662         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 663         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 664
 665         for (BigInteger dpId : targetDpns) {
 666             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 667                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 668
 669             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 670                                /*writeTx*/null);
 671         }
 672     }
 673
 674
 675     /*
 676      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 677      */
 678     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 679                                                final VrfEntry vrfEntry, long vpnTag) {
 680         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 681         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 682             && vrfEntry.getRoutePaths().size() == 1);
 683         String destination = vrfEntry.getDestPrefix();
 684         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 685         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 686
 687         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 688         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 689         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 690         if (interVpnLink.getState().or(State.Error) != State.Active) {
 691             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 692                 destination, nextHop, interVpnLinkName);
 693             return;
 694         }
 695
 696         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 697         if (!optOtherEndpointLportTag.isPresent()) {
 698             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 699                 vpnUuid, interVpnLinkName);
 700             return;
 701         }
 702
 703         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 704         if (targetDpns.isEmpty()) {
 705             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 706                 vpnUuid, interVpnLinkName);
 707             return;
 708         }
 709
 710         String[] values = destination.split("/");
 711         String destPrefixIpAddress = values[0];
 712         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 713
 714         List<MatchInfo> matches = new ArrayList<>();
 715         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 716         matches.add(MatchEthernetType.IPV4);
 717
 718         if (prefixLength != 0) {
 719             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 720         }
 721
 722         List<Instruction> instructions =
 723             Arrays.asList(new InstructionWriteMetadata(
 724                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 725                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 726                             .L3VPN_SERVICE_INDEX)),
 727                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 728                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 729
 730         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 731         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 732         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 733             COOKIE_VM_FIB_TABLE, matches, instructions);
 734
 735         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 736             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 737
 738         for (BigInteger dpId : targetDpns) {
 739
 740             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 741                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 742                 dpId, interVpnLink.getInterVpnLinkName());
 743
 744             mdsalManager.installFlow(dpId, flowEntity);
 745         }
 746     }
 747
 748     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 749         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 750         String localNextHopIP = vrfEntry.getDestPrefix();
 751         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 752         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 753         if (localNextHopInfo == null) {
 754             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 755             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 756                     vpnName, usedRds, localNextHopIP);
 757             if (LOG.isDebugEnabled()) {
 758                 LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 759                         vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 760             }
 761             boolean localNextHopSeen = false;
 762             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 763             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 764                 String ipPrefix;
 765                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 766                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 767                 } else {
 768                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 769                 }
 770                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 771                     ipPrefix);
 772                 if (localNextHopInfoLocal != null) {
 773                     localNextHopSeen = true;
 774                     BigInteger dpnId =
 775                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 776                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 777                     returnLocalDpnId.add(dpnId);
 778                 }
 779             }
 780             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 781                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 782                 if (optionalLabel.isPresent()) {
 783                     Long label = optionalLabel.get();
 784                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 785                     synchronized (label.toString().intern()) {
 786                         LabelRouteInfo lri = getLabelRouteInfo(label);
 787                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 788                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 789                                     fibUtil.getVpnInstanceOpData(rd);
 790                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 791                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 792                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 793                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 794                                     localNextHopIP = lri.getPrefix();
 795                                 } else {
 796                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 797                                     localNextHopIP = lri.getPrefix();
 798                                 }
 799                             }
 800                             if (localNextHopInfo != null) {
 801                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 802                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 803                                 if (vpnExtraRoutes.isEmpty()) {
 804                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 805                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 806                                     returnLocalDpnId.add(dpnId);
 807                                 } else {
 808                                     for (Routes extraRoutes : vpnExtraRoutes) {
 809                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 810                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 811                                         returnLocalDpnId.add(dpnId);
 812                                     }
 813                                 }
 814                             }
 815                         }
 816                     }
 817                 }
 818             }
 819             if (returnLocalDpnId.isEmpty()) {
 820                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 821             }
 822         } else {
 823             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 824                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 825             if (dpnId != null) {
 826                 returnLocalDpnId.add(dpnId);
 827             }
 828         }
 829         return returnLocalDpnId;
 830     }
 831
 832     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 833                                                 final Long vpnId, final String rd,
 834                                                 final VrfEntry vrfEntry,
 835                                                 Routes routes, List<Routes> vpnExtraRoutes,
 836                                                 int etherType) {
 837         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 838         if (localNextHopInfo != null) {
 839             long groupId;
 840             long localGroupId;
 841             final BigInteger dpnId = localNextHopInfo.getDpnId();
 842             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 843                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 844                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 845                 return dpnId;
 846             }
 847             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 848                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 849                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 850                 return dpnId;
 851             }
 852             if (!isVpnPresentInDpn(rd, dpnId)) {
 853                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 854                         vpnId, rd, dpnId.toString());
 855                 return BigInteger.ZERO;
 856             }
 857             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 858             String prefix = vrfEntry.getDestPrefix();
 859             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 860             //The loadbalancing group is created only if the extra route has multiple nexthops
 861             //to avoid loadbalancing the discovered routes
 862             if (vpnExtraRoutes != null && routes != null) {
 863                 if (vpnExtraRoutes.size() > 1) {
 864                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 865                     localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix());
 866                 } else {
 867                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 868                     localGroupId = groupId;
 869                 }
 870             } else {
 871                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 872                         gwMacAddress);
 873                 localGroupId = groupId;
 874             }
 875             if (groupId == FibConstants.INVALID_GROUP_ID) {
 876                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 877                         prefix, rd, interfaceName, dpnId.toString());
 878                 return BigInteger.ZERO;
 879             }
 880             final List<InstructionInfo> instructions = Collections.singletonList(
 881                     new InstructionApplyActions(
 882                             Collections.singletonList(new ActionGroup(groupId))));
 883             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 884                     new InstructionApplyActions(
 885                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId))));
 886             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 887             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 888             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 889             jobCoordinator.enqueueJob(jobKey,
 890                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 891                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 892                             NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
 893                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 894                         optLabel.ifPresent(label -> {
 895                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 896                                 LOG.debug(
 897                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 898                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 899                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 900                                         nextHopAddressList);
 901                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 902                                         NwConstants.ADD_FLOW, tx);
 903                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 904                             } else {
 905                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 906                                                 + "route. LFib and Terminating table entries will not be created.",
 907                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 908                             }
 909                         });
 910                     }
 911                 })));
 912             return dpnId;
 913         }
 914         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 915                 vpnName);
 916         return BigInteger.ZERO;
 917     }
 918
 919     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 920         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 921         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 922         if (dpnInVpn.isPresent()) {
 923             return true;
 924         }
 925         return false;
 926     }
 927
 928     private LabelRouteInfo getLabelRouteInfo(Long label) {
 929         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 930             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 931         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 932         if (opResult.isPresent()) {
 933             return opResult.get();
 934         }
 935         return null;
 936     }
 937
 938     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName,
 939             TypedWriteTransaction<Operational> tx) {
 940         if (lri == null) {
 941             return true;
 942         }
 943
 944         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 945         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 946             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 947
 948         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 949             ? lri.getVpnInstanceList() : new ArrayList<>();
 950         if (vpnInstancesList.contains(vpnInstanceName)) {
 951             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 952             vpnInstancesList.remove(vpnInstanceName);
 953         }
 954         if (vpnInstancesList.isEmpty()) {
 955             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 956             if (tx != null) {
 957                 tx.delete(lriId);
 958             } else {
 959                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 960             }
 961             return true;
 962         } else {
 963             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 964             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 965             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 966         }
 967         return false;
 968     }
 969
 970     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 971                                       TypedWriteTransaction<Configuration> tx) {
 972         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 973
 974         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 975
 976         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 977             dpId, label, groupId);
 978     }
 979
 980     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 981                                                 TypedWriteTransaction<Configuration> tx) {
 982         List<MatchInfo> mkMatches = new ArrayList<>();
 983
 984         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 985             destDpId, label, actionsInfos);
 986
 987         // Matching metadata
 988         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 989         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 990
 991         List<InstructionInfo> mkInstructions = new ArrayList<>();
 992         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 993
 994         FlowEntity terminatingServiceTableFlowEntity =
 995             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 996             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 997                 String.format("%s:%d", "TST Flow Entry ", label),
 998             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 999
1000         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1001
1002         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1003
1004         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1005         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1006             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1007             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1008             .child(Flow.class, flowKey).build();
1009         tx.put(flowInstanceId, flowbld.build(), CREATE_MISSING_PARENTS);
1010     }
1011
1012     private void removeTunnelTableEntry(BigInteger dpId, long label, TypedWriteTransaction<Configuration> tx) {
1013         FlowEntity flowEntity;
1014         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1015         List<MatchInfo> mkMatches = new ArrayList<>();
1016         // Matching metadata
1017         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1018         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1019             NwConstants.INTERNAL_TUNNEL_TABLE,
1020             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1021             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1022             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1023         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1024         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1025         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1026             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1027             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1028
1029         tx.delete(flowInstanceId);
1030         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1031     }
1032
1033     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1034         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1035         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1036         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1037         boolean shouldUpdateNonEcmpLocalNextHop = true;
1038         if (localNextHopInfo == null) {
1039             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1040             if (usedRds.size() > 1) {
1041                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1042                         vrfEntry.getDestPrefix(), vpnName, rd);
1043                 return returnLocalDpnId;
1044             }
1045             String vpnRd = !usedRds.isEmpty() ? usedRds.get(0) : rd;
1046             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1047             //in the vpn
1048             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1049                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1050             if (extraRouteOptional.isPresent()) {
1051                 Routes extraRoute = extraRouteOptional.get();
1052                 String ipPrefix;
1053                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1054                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1055                 } else {
1056                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1057                 }
1058                 if (extraRoute.getNexthopIpList().size() > 1) {
1059                     shouldUpdateNonEcmpLocalNextHop = false;
1060                 }
1061                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1062                 if (localNextHopInfo != null) {
1063                     String localNextHopIP = localNextHopInfo.getIpAddress();
1064                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1065                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1066                     if (!dpnId.equals(BigInteger.ZERO)) {
1067                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1068                         nextHopManager.deleteLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix());
1069                         returnLocalDpnId.add(dpnId);
1070                     }
1071                 } else {
1072                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1073                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1074                 }
1075             }
1076
1077             if (localNextHopInfo == null) {
1078                 /* Imported VRF entry */
1079                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1080                 if (optionalLabel.isPresent()) {
1081                     Long label = optionalLabel.get();
1082                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1083                     LabelRouteInfo lri = getLabelRouteInfo(label);
1084                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1085                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1086                         prefixBuilder.setDpnId(lri.getDpnId());
1087                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1088                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1089                         if (!dpnId.equals(BigInteger.ZERO)) {
1090                             returnLocalDpnId.add(dpnId);
1091                         }
1092                     }
1093                 }
1094             }
1095
1096         } else {
1097             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1098             String localNextHopIP = localNextHopInfo.getIpAddress();
1099             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1100                     shouldUpdateNonEcmpLocalNextHop);
1101             if (!dpnId.equals(BigInteger.ZERO)) {
1102                 returnLocalDpnId.add(dpnId);
1103             }
1104         }
1105
1106         return returnLocalDpnId;
1107     }
1108
1109     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1110             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1111             boolean shouldUpdateNonEcmpLocalNextHop) {
1112         if (localNextHopInfo != null) {
1113             final BigInteger dpnId = localNextHopInfo.getDpnId();
1114             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1115                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1116                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1117                 return dpnId;
1118             }
1119             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1120                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1121                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1122                 return dpnId;
1123             }
1124
1125             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1126                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1127                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1128                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1129                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1130                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1131                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1132                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1133                                         NwConstants.DEL_FLOW, tx);
1134                                 removeTunnelTableEntry(dpnId, label, tx);
1135                             });
1136                         }
1137                     }
1138                 })));
1139             //TODO: verify below adjacency call need to be optimized (?)
1140             //In case of the removal of the extra route, the loadbalancing group is updated
1141             if (shouldUpdateNonEcmpLocalNextHop) {
1142                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1143             }
1144             return dpnId;
1145         }
1146         return BigInteger.ZERO;
1147     }
1148
1149     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1150             final VrfEntry vrfEntry, TypedWriteTransaction<Configuration> tx) {
1151         if (tx == null) {
1152             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1153                 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG,
1154                 "Error creating remote FIB entry");
1155             return;
1156         }
1157
1158         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1159         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1160                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1161
1162         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1163         if (adjacencyResults.isEmpty()) {
1164             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1165                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1166             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1167             return;
1168         }
1169
1170         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1171         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1172                 vpnName, usedRds, vrfEntry.getDestPrefix());
1173         // create loadbalancing groups for extra routes only when the extra route is present behind
1174         // multiple VMs
1175         if (!vpnExtraRoutes.isEmpty()) {
1176             List<InstructionInfo> instructions = new ArrayList<>();
1177             // Obtain the local routes for this particular dpn.
1178             java.util.Optional<Routes> routes = vpnExtraRoutes
1179                     .stream()
1180                     .filter(route -> {
1181                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1182                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1183                         if (prefixToInterface == null) {
1184                             return false;
1185                         }
1186                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1187                     }).findFirst();
1188             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1189                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1190             if (groupId == FibConstants.INVALID_GROUP_ID) {
1191                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1192                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1193                 return;
1194             }
1195             List<ActionInfo> actionInfos =
1196                     Collections.singletonList(new ActionGroup(groupId));
1197             instructions.add(new InstructionApplyActions(actionInfos));
1198             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1199                     NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1200         } else {
1201             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry,
1202                 TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null);
1203         }
1204
1205         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1206     }
1207
1208     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1209     /* Get interface info from prefix to interface mapping;
1210         Use the interface info to get the corresponding vpn interface op DS entry,
1211         remove the adjacency corresponding to this fib entry.
1212         If adjacency removed is the last adjacency, clean up the following:
1213          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1214          - prefix to interface entry
1215          - vpn interface op DS
1216      */
1217         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1218         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1219         if (prefixInfo == null) {
1220             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1221             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1222             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1223             if (extraRoute != null) {
1224                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1225                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1226                     if (nextHopIp != null) {
1227                         String ipPrefix;
1228                         if (isIpv4Address(nextHopIp)) {
1229                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1230                         } else {
1231                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1232                         }
1233                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1234                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1235                     }
1236                 }
1237             }
1238             if (prefixInfo == null) {
1239                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1240                 if (optionalLabel.isPresent()) {
1241                     Long label = optionalLabel.get();
1242                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1243                     LabelRouteInfo lri = getLabelRouteInfo(label);
1244                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1245                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1246                         prefixBuilder.setDpnId(lri.getDpnId());
1247                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1248                         prefixBuilder.setIpAddress(lri.getPrefix());
1249                         prefixInfo = prefixBuilder.build();
1250                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1251                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1252                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1253                     }
1254                 }
1255             }
1256         } else {
1257             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1258         }
1259     }
1260
1261     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1262                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1263
1264         if (prefixInfo == null) {
1265             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1266                     vrfEntry.getDestPrefix(), vpnId, rd);
1267             return; //Don't have any info for this prefix (shouldn't happen); need to return
1268         }
1269
1270         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1271             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1272                     vrfEntry.getDestPrefix(), vpnId, rd);
1273             return;
1274         }
1275
1276         String ifName = prefixInfo.getVpnInterfaceName();
1277         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1278             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1279     }
1280
1281     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1282         Prefixes prefixInfo;
1283         Long vpnId;
1284         String rd;
1285         VrfEntry vrfEntry;
1286         Routes extraRoute;
1287
1288         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1289                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1290             this.prefixInfo = prefixInfo;
1291             this.vpnId = vpnId;
1292             this.rd = rd;
1293             this.vrfEntry = vrfEntry;
1294             this.extraRoute = extraRoute;
1295         }
1296
1297         @Override
1298         public List<ListenableFuture<Void>> call() {
1299             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1300             // to call the respective helpers.
1301             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
1302                 //First Cleanup LabelRouteInfo
1303                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1304                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1305                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1306                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1307                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1308                         synchronized (label.toString().intern()) {
1309                             LabelRouteInfo lri = getLabelRouteInfo(label);
1310                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1311                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1312                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1313                                         fibUtil.getVpnInstanceOpData(rd);
1314                                 String vpnInstanceName = "";
1315                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1316                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1317                                 }
1318                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1319                                 if (lriRemoved) {
1320                                     String parentRd = lri.getParentVpnRd();
1321                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1322                                             parentRd, vrfEntry.getDestPrefix()));
1323                                 }
1324                             } else {
1325                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1326                                         rd, vrfEntry.getDestPrefix()));
1327                             }
1328                         }
1329                     });
1330                 }
1331                 String ifName = prefixInfo.getVpnInterfaceName();
1332                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1333                 String vpnName = null;
1334
1335                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1336                     // Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1337                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1338                     if (vpnNameOpt.isPresent()) {
1339                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1340                     }
1341                 }
1342                 if (optVpnName.isPresent()) {
1343                     vpnName = optVpnName.get();
1344                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = tx
1345                             .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get();
1346                     if (opVpnInterface.isPresent()) {
1347                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1348                         if (vpnId != associatedVpnId) {
1349                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1350                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1351                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1352                             return;
1353                         } else {
1354                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1355                                     vrfEntry.getDestPrefix(), associatedVpnId);
1356                         }
1357                     }
1358                 }
1359                 if (extraRoute != null) {
1360                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1361                     //Only one used Rd present in case of removal event
1362                     String usedRd = usedRds.get(0);
1363                     if (optVpnName.isPresent()) {
1364                         tx.delete(BaseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1365                                         vrfEntry.getDestPrefix()));
1366                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx ->
1367                             configTx.delete(VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix())));
1368                         nextHopManager.removeNextHopPointer(nextHopManager
1369                                 .getRemoteSelectGroupKey(vpnId, vrfEntry.getDestPrefix()));
1370                         nextHopManager.removeNextHopPointer(nextHopManager
1371                                 .getLocalSelectGroupKey(vpnId, vrfEntry.getDestPrefix()));
1372                     }
1373                 }
1374                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1375             }));
1376         }
1377     }
1378
1379     /**
1380      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1381      * Remove Adjacency from VPNInterfaceOpData.
1382      * if Adjacency != primary.
1383      * if Adjacency == primary , then mark it for deletion.
1384      * Remove entire VPNinterfaceOpData Entry.
1385      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1386      * @param vrfEntry - VrfEntry removed
1387      * @param ifName - Interface name from VRFentry
1388      * @param vpnName - VPN name of corresponding VRF
1389      * @param tx - ReadWrite Tx
1390      */
1391     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1392                                                           TypedReadWriteTransaction<Operational> tx)
1393             throws ExecutionException, InterruptedException {
1394         InstanceIdentifier<Adjacency> adjacencyIid =
1395                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1396         Optional<Adjacency> adjacencyOptional = tx.read(adjacencyIid).get();
1397         if (adjacencyOptional.isPresent()) {
1398             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1399                 tx.delete(FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1400             } else {
1401                 tx.merge(adjacencyIid,
1402                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1403             }
1404         }
1405
1406         Optional<AdjacenciesOp> optAdjacencies = tx.read(FibUtil.getAdjListPathOp(ifName, vpnName)).get();
1407
1408         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1409             return;
1410         }
1411
1412         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1413                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1414                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1415                         && adjacency.isMarkedForDeletion() != null
1416                         && adjacency.isMarkedForDeletion()
1417         )) {
1418             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1419             tx.delete(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1420         }
1421     }
1422
1423     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1424         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1425         final String rd = vrfTableKey.getRouteDistinguisher();
1426         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1427         if (vpnInstance == null) {
1428             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1429             return;
1430         }
1431         final Collection<VpnToDpnList> vpnToDpnList;
1432         if (vrfEntry.getParentVpnRd() != null
1433                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1434             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1435             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1436             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1437                     vpnInstance.getVpnToDpnList();
1438             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1439                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1440         } else {
1441             vpnToDpnList = vpnInstance.getVpnToDpnList();
1442         }
1443
1444         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1445         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1446         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1447         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1448         if (subnetRoute != null) {
1449             long elanTag = subnetRoute.getElantag();
1450             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1451                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1452             if (vpnToDpnList != null) {
1453                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1454                     () -> Collections.singletonList(
1455                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1456                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1457
1458                                 baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(),
1459                                     vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW,
1460                                     TransactionAdapter.toWriteTransaction(tx), null);
1461                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1462                                     optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1463                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1464                                 }
1465
1466                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1467                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1468                             }
1469                         })));
1470             }
1471             optionalLabel.ifPresent(label -> {
1472                 synchronized (label.toString().intern()) {
1473                     LabelRouteInfo lri = getLabelRouteInfo(label);
1474                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1475                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1476                                 fibUtil.getVpnInstanceOpData(rd);
1477                         String vpnInstanceName = "";
1478                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1479                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1480                         }
1481                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1482                         if (lriRemoved) {
1483                             String parentRd = lri.getParentVpnRd();
1484                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1485                                     parentRd, vrfEntry.getDestPrefix()));
1486                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1487                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1488                         }
1489                     } else {
1490                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1491                                 rd, vrfEntry.getDestPrefix()));
1492                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1493                                 label, rd, vrfEntry.getDestPrefix());
1494                     }
1495                 }
1496             });
1497             return;
1498         }
1499
1500         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1501             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1502         if (vpnToDpnList != null) {
1503             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1504                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1505             String jobKey;
1506             Optional<Routes> extraRouteOptional;
1507             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1508             if (usedRds != null && !usedRds.isEmpty()) {
1509                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1510                 if (usedRds.size() > 1) {
1511                     LOG.error("The extra route prefix is still present in some DPNs");
1512                     return ;
1513                 } else {
1514                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1515                     //is not present in any other DPN
1516                     extraRouteOptional = VpnExtraRouteHelper
1517                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1518                 }
1519             } else {
1520                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1521                 extraRouteOptional = Optional.absent();
1522             }
1523
1524             jobCoordinator.enqueueJob(jobKey,
1525                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1526                     if (localDpnIdList.size() <= 0) {
1527                         for (VpnToDpnList curDpn : vpnToDpnList) {
1528                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1529                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1530                                 TransactionAdapter.toWriteTransaction(tx));
1531                         }
1532                     } else {
1533                         for (BigInteger localDpnId : localDpnIdList) {
1534                             for (VpnToDpnList curDpn : vpnToDpnList) {
1535                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1536                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1537                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional,
1538                                         TransactionAdapter.toWriteTransaction(tx));
1539                                 }
1540                             }
1541                         }
1542                     }
1543                 })), MAX_RETRIES);
1544         }
1545
1546         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1547         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1548         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1549
1550         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1551         // of the interVpnLink.
1552         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1553         if (optVpnUuid.isPresent()) {
1554             String vpnUuid = optVpnUuid.get();
1555             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1556                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1557                 if (optInterVpnLink.isPresent()) {
1558                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1559                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1560                         // This is route that points to the other endpoint of an InterVpnLink
1561                         // In that case, we should look for the FIB table pointing to
1562                         // LPortDispatcher table and remove it.
1563                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1564                     }
1565                 }
1566             });
1567         }
1568
1569     }
1570
1571     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1572                                     int addOrRemove, TypedWriteTransaction<Configuration> tx) {
1573         if (tx == null) {
1574             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1575                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1576                 "Error making LFIB table entry");
1577             return;
1578         }
1579
1580         List<MatchInfo> matches = new ArrayList<>();
1581         matches.add(MatchEthernetType.MPLS_UNICAST);
1582         matches.add(new MatchMplsLabel(label));
1583
1584         // Install the flow entry in L3_LFIB_TABLE
1585         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1586
1587         FlowEntity flowEntity;
1588         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1589             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1590         Flow flow = flowEntity.getFlowBuilder().build();
1591         String flowId = flowEntity.getFlowId();
1592         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1593         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1594         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1595             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1596             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1597
1598         if (addOrRemove == NwConstants.ADD_FLOW) {
1599             tx.put(flowInstanceId, flow, CREATE_MISSING_PARENTS);
1600         } else {
1601             tx.delete(flowInstanceId);
1602         }
1603
1604         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1605             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1606     }
1607
1608     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1609                                     final FutureCallback<List<Void>> callback) {
1610         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1611         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1612             () -> {
1613                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1614                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1615                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1616                         LogicalDatastoreType.CONFIGURATION, id);
1617                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1618                 if (!vrfTable.isPresent()) {
1619                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1620                     if (callback != null) {
1621                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1622                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1623                     }
1624                     return futures;
1625                 }
1626                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1627                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1628                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1629                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1630                             if (subnetRoute != null) {
1631                                 long elanTag = subnetRoute.getElantag();
1632                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1633                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1634                                         tx);
1635                                 continue;
1636                             }
1637                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1638                             if (routerInt != null) {
1639                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1640                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1641                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1642                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1643                                         NwConstants.ADD_FLOW);
1644                                 continue;
1645                             }
1646                             //Handle local flow creation for imports
1647                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1648                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1649                                 if (optionalLabel.isPresent()) {
1650                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1651                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1652                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1653                                         if (lri.getDpnId().equals(dpnId)) {
1654                                             try {
1655                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1656                                                         vrfEntry.getDestPrefix());
1657                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1658                                             } catch (IllegalArgumentException ex) {
1659                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1660                                                         vrfEntry.getDestPrefix());
1661                                             }
1662                                             continue;
1663                                         }
1664                                     }
1665                                 }
1666                             }
1667                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1668                                     vrfEntry, dpnId);
1669                             if (shouldCreateRemoteFibEntry) {
1670                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1671                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1672                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1673                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1674                                             vrfTable.get().getRouteDistinguisher(), vrfEntry,
1675                                             TransactionAdapter.toWriteTransaction(tx), txnObjects);
1676                                 } else {
1677                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1678                                             vrfEntry, tx);
1679                                 }
1680                             }
1681                         }
1682                     }));
1683                     if (callback != null) {
1684                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1685                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1686                     }
1687                 }
1688                 return futures;
1689             });
1690     }
1691
1692     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1693                                             final String localNextHopIp, final String remoteNextHopIp) {
1694         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1695             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1696         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1697         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1698         List<SubTransaction> txnObjects =  new ArrayList<>();
1699         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1700         if (vrfTable.isPresent()) {
1701             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1702                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1703                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1704                         vrfTable.get().getVrfEntry().stream()
1705                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1706                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1707                                 rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects));
1708                     }
1709                 })));
1710         }
1711     }
1712
1713     public void manageRemoteRouteOnDPN(final boolean action,
1714                                        final BigInteger localDpnId,
1715                                        final long vpnId,
1716                                        final String rd,
1717                                        final String destPrefix,
1718                                        final String destTepIp,
1719                                        final long label) {
1720         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1721
1722         if (vpnInstance == null) {
1723             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1724             return;
1725         }
1726
1727         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1728             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1729                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1730                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1731                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1732                     if (vrfEntry == null) {
1733                         return;
1734                     }
1735                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1736                             action, localDpnId, vpnId, rd, destPrefix);
1737                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1738                     VrfEntry modVrfEntry;
1739                     if (routePathList == null || routePathList.isEmpty()) {
1740                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1741                                 Collections.singletonList(destTepIp),
1742                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1743                     } else {
1744                         modVrfEntry = vrfEntry;
1745                     }
1746
1747                     if (action) {
1748                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1749                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1750                                 modVrfEntry, tx);
1751                     } else {
1752                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1753                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1754                                 vrfEntry.getDestPrefix());
1755                         if (usedRds.size() > 1) {
1756                             LOG.debug("The extra route prefix is still present in some DPNs");
1757                             return;
1758                         }
1759                         //Is this fib route an extra route? If yes, get the nexthop which would be
1760                         //an adjacency in the vpn
1761                         Optional<Routes> extraRouteOptional = Optional.absent();
1762                         if (usedRds.size() != 0) {
1763                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1764                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1765                                     usedRds.get(0), vrfEntry.getDestPrefix());
1766                         }
1767                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1768                                 extraRouteOptional, TransactionAdapter.toWriteTransaction(tx));
1769                     }
1770                 }
1771             })));
1772     }
1773
1774     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1775                                  final FutureCallback<List<Void>> callback) {
1776         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1777         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1778             () -> {
1779                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1780                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1781                 List<SubTransaction> txnObjects =  new ArrayList<>();
1782                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1783                         LogicalDatastoreType.CONFIGURATION, id);
1784                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1785                 if (vrfTable.isPresent()) {
1786                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1787                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1788                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1789                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1790                                 /* Handle subnet routes here */
1791                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1792                                 if (subnetRoute != null && !fibUtil
1793                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1794                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1795                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1796                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1797                                             NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null);
1798                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1799                                     if (routePaths != null) {
1800                                         for (RoutePaths routePath : routePaths) {
1801                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1802                                                     DEFAULT_FIB_FLOW_PRIORITY,
1803                                                     NwConstants.DEL_FLOW, tx);
1804                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1805                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1806                                                     vrfEntry.getDestPrefix());
1807                                         }
1808                                     }
1809                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1810                                             NwConstants.DEL_FLOW, tx);
1811                                     continue;
1812                                 }
1813                                 // ping responder for router interfaces
1814                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1815                                 if (routerInt != null) {
1816                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1817                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1818                                             routerInt.getMacAddress());
1819                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1820                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1821                                             NwConstants.DEL_FLOW);
1822                                     continue;
1823                                 }
1824
1825                                 //Handle local flow deletion for imports
1826                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1827                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1828                                     if (optionalLabel.isPresent()) {
1829                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1830                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1831                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1832                                                 lri) && lri.getDpnId().equals(dpnId)) {
1833                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1834                                         }
1835                                     }
1836                                 }
1837
1838                                 // Passing null as we don't know the dpn
1839                                 // to which prefix is attached at this point
1840                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1841                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1842                                 Optional<Routes> extraRouteOptional;
1843                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1844                                 //an adjacency in the vpn
1845                                 if (usedRds != null && !usedRds.isEmpty()) {
1846                                     if (usedRds.size() > 1) {
1847                                         LOG.error("The extra route prefix is still present in some DPNs");
1848                                         return;
1849                                     } else {
1850                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1851                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1852
1853                                     }
1854                                 } else {
1855                                     extraRouteOptional = Optional.absent();
1856                                 }
1857                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1858                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1859                                         vrfTable.get().key(), vrfEntry, extraRouteOptional,
1860                                         TransactionAdapter.toWriteTransaction(tx), txnObjects);
1861                                 } else {
1862                                     if (subnetRoute == null || !fibUtil
1863                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1864                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1865                                             vrfTable.get().key(), vrfEntry, extraRouteOptional,
1866                                             TransactionAdapter.toWriteTransaction(tx));
1867                                     }
1868                                 }
1869                             }
1870                         }));
1871                     }
1872                     if (callback != null) {
1873                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1874                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1875                     }
1876                 } else {
1877                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1878                 }
1879                 return futures;
1880             });
1881
1882     }
1883
1884     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1885                                            final String localNextHopIp, final String remoteNextHopIp) {
1886         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1887                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1888             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1889         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1890         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1891         List<SubTransaction> txnObjects =  new ArrayList<>();
1892         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1893         if (vrfTable.isPresent()) {
1894             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1895                 () -> {
1896                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1897                         return Collections.singletonList(
1898                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1899                                 tx -> vrfTable.get().getVrfEntry().stream()
1900                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1901                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1902                                         remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx),
1903                                         txnObjects))));
1904                     }
1905                 });
1906         }
1907     }
1908
1909     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1910         InstanceIdentifierBuilder<VrfTables> idBuilder =
1911             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1912         return idBuilder.build();
1913     }
1914
1915     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1916         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1917                 .FLOWID_SEPARATOR + nextHop;
1918     }
1919
1920     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1921         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1922                 + tableMiss + FLOWID_PREFIX;
1923     }
1924
1925     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1926         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1927             .child(VrfTables.class, new VrfTablesKey(rd))
1928             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1929         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1930         if (vrfEntry.isPresent()) {
1931             return vrfEntry.get();
1932         }
1933         return null;
1934     }
1935
1936     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1937                                              final String vpnName,
1938                                              final VrfEntry vrfEntry) {
1939         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1940
1941         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1942         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1943
1944         if (targetDpns.isEmpty()) {
1945             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1946             return;
1947         }
1948
1949         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1950         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1951
1952         // delete from FIB
1953         //
1954         optNextHop.ifPresent(nextHop -> {
1955             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1956             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1957             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1958                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1959
1960             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1961             for (BigInteger dpId : targetDpns) {
1962                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1963                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1964
1965                 mdsalManager.removeFlow(dpId, flow);
1966             }
1967         });
1968
1969         // delete from LFIB
1970         //
1971         optLabel.ifPresent(label -> {
1972             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1973
1974             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1975                 for (BigInteger dpId : targetDpns) {
1976                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1977                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1978                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1979                             tx);
1980                 }
1981             }), LOG, "Error removing flows");
1982         });
1983     }
1984
1985     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1986             List<String> nextHopAddressList, LabelRouteInfo lri) {
1987         return lri != null && lri.getPrefix().equals(prefix)
1988                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1989     }
1990
1991     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1992         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1993             return true;
1994         }
1995
1996         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1997         if (prefix != null) {
1998             BigInteger prefixDpnId = prefix.getDpnId();
1999             if (dpnId.equals(prefixDpnId)) {
2000                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2001                         vrfEntry, dpnId);
2002                 return false;
2003             }
2004         }
2005         return true;
2006     }
2007 }