vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Arrays;
  20 import java.util.Collection;
  21 import java.util.Collections;
  22 import java.util.List;
  23 import java.util.concurrent.Callable;
  24
  25 import javax.annotation.PostConstruct;
  26 import javax.inject.Inject;
  27 import javax.inject.Singleton;
  28 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  29 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  32 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  33 import org.opendaylight.genius.mdsalutil.ActionInfo;
  34 import org.opendaylight.genius.mdsalutil.FlowEntity;
  35 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  36 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  37 import org.opendaylight.genius.mdsalutil.MatchInfo;
  38 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  47 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  48 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  49 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  50 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  51 import org.opendaylight.genius.utils.ServiceIndex;
  52 import org.opendaylight.genius.utils.batching.SubTransaction;
  53 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  54 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  55 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  56 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  58 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  59 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
  92 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  93 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97
  98 @Singleton
  99 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 100     implements AutoCloseable {
 101
 102     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 103     private static final String FLOWID_PREFIX = "L3.";
 104     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 105     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 106     private static final int LFIB_INTERVPN_PRIORITY = 15;
 107     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 108
 109     private final DataBroker dataBroker;
 110     private final IMdsalApiManager mdsalManager;
 111     private final NexthopManager nextHopManager;
 112     private final IdManagerService idManager;
 113     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 114     private final BaseVrfEntryHandler baseVrfEntryHandler;
 115     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 116
 117     protected static boolean isOpenStackVniSemanticsEnforced;
 118
 119     @Inject
 120     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 121                             final NexthopManager nexthopManager, final IdManagerService idManager,
 122                             final IElanService elanManager,
 123                             final BaseVrfEntryHandler vrfEntryHandler,
 124                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 125                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler) {
 126         super(VrfEntry.class, VrfEntryListener.class);
 127         this.dataBroker = dataBroker;
 128         this.mdsalManager = mdsalApiManager;
 129         this.nextHopManager = nexthopManager;
 130         this.idManager = idManager;
 131         this.baseVrfEntryHandler = vrfEntryHandler;
 132         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 133         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 134
 135         isOpenStackVniSemanticsEnforced = elanManager.isOpenStackVniSemanticsEnforced();
 136     }
 137
 138     @Override
 139     @PostConstruct
 140     public void init() {
 141         LOG.info("{} init", getClass().getSimpleName());
 142         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 143     }
 144
 145     @Override
 146     protected VrfEntryListener getDataTreeChangeListener() {
 147         return VrfEntryListener.this;
 148     }
 149
 150     @Override
 151     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 152         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 153     }
 154
 155     @Override
 156     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 157         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 158         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 159         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 160                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 161         addFibEntries(identifier, vrfEntry, rd);
 162         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 163                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 164     }
 165
 166     //This method is temporary. Eventually Factory design pattern will be used to get
 167     // right VrfEntryhandle and invoke its methods.
 168     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 169         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 170             LOG.info("EVPN flows need to be programmed.");
 171             EvpnVrfEntryHandler evpnVrfEntryHandler =
 172                     new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler, nextHopManager);
 173             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 174             return;
 175         }
 176         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
 177         if (routerInt != null) {
 178             // ping responder for router interfaces
 179             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 180             return;
 181         }
 182         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 183             createFibEntries(identifier, vrfEntry);
 184             return;
 185         }
 186         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 187             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 188             return;
 189         }
 190     }
 191
 192     @Override
 193     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 194         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 195         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 196         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 197                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 198         removeFibEntries(identifier, vrfEntry, rd);
 199         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 200             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 201     }
 202
 203     //This method is temporary. Eventually Factory design pattern will be used to get
 204     // right VrfEntryhandle and invoke its methods.
 205     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 206         if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Vxlan)) {
 207             LOG.info("EVPN flows to be deleted");
 208             EvpnVrfEntryHandler evpnVrfEntryHandler =
 209                     new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler, nextHopManager);
 210             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 211             return;
 212         }
 213         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
 214         if (routerInt != null) {
 215             // ping responder for router interfaces
 216             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 217             return;
 218         }
 219         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 220             deleteFibEntries(identifier, vrfEntry);
 221             return;
 222         }
 223         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 224             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 225             return;
 226         }
 227     }
 228
 229     @Override
 230     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 231         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 232         if (original.equals(update)) {
 233             return;
 234         }
 235         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 236         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {}",
 237             rd, update.getDestPrefix(), update.getRoutePaths());
 238         // Handle BGP Routes first
 239         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 240             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 241             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 242                 rd, update.getDestPrefix(), update.getRoutePaths());
 243             return;
 244         }
 245
 246         // Handle Vpn Interface driven Routes next (ie., STATIC and LOCAL)
 247         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(update.getOrigin()))) {
 248             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 249             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 250             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 251
 252             // If original VRF Entry had nexthop null , but update VRF Entry
 253             // has nexthop , route needs to be created on remote Dpns
 254             if (((originalRoutePath == null) || (originalRoutePath.isEmpty())
 255                 && (updateRoutePath != null) && (!updateRoutePath.isEmpty()))) {
 256                 // TODO(vivek): Though ugly, Not handling this code now, as each
 257                 // tep add event will invoke flow addition
 258                 LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 259                     update.getDestPrefix());
 260                 return;
 261             }
 262
 263             // If original VRF Entry had valid nexthop , but update VRF Entry
 264             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 265             if (((updateRoutePath == null) || (updateRoutePath.isEmpty())
 266                 && (originalRoutePath != null) && (!originalRoutePath.isEmpty()))) {
 267                 LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 268                     update.getDestPrefix());
 269                 return;
 270             }
 271             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 272             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 273             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 274             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 275             nextHopsRemoved.parallelStream()
 276                     .forEach(nextHopRemoved -> FibUtil.updateUsedRdAndVpnToExtraRoute(
 277                              writeOperTxn, dataBroker, nextHopRemoved, rd,
 278                              update.getDestPrefix()));
 279             writeOperTxn.submit();
 280             createFibEntries(identifier, update);
 281             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 282                 rd, update.getDestPrefix(), update.getRoutePaths());
 283             return;
 284         }
 285
 286         /* Handl all other route origins */
 287         createFibEntries(identifier, update);
 288
 289         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 290             rd, update.getDestPrefix(), update.getRoutePaths());
 291     }
 292
 293     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 294         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 295         List<SubTransaction> txnObjects =  new ArrayList<>();
 296         final VpnInstanceOpDataEntry vpnInstance =
 297                 FibUtil.getVpnInstance(dataBroker, vrfTableKey.getRouteDistinguisher());
 298         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 299         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 300                 + " has null vpnId!");
 301         final Collection<VpnToDpnList> vpnToDpnList;
 302         if (vrfEntry.getParentVpnRd() != null
 303                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 304             VpnInstanceOpDataEntry parentVpnInstance = FibUtil.getVpnInstance(dataBroker, vrfEntry.getParentVpnRd());
 305             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 306                 vpnInstance.getVpnToDpnList();
 307         } else {
 308             vpnToDpnList = vpnInstance.getVpnToDpnList();
 309         }
 310         final Long vpnId = vpnInstance.getVpnId();
 311         final String rd = vrfTableKey.getRouteDistinguisher();
 312         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 313         if (subnetRoute != null) {
 314             final long elanTag = subnetRoute.getElantag();
 315             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 316                     rd, vrfEntry.getDestPrefix(), elanTag);
 317             if (vpnToDpnList != null) {
 318                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 319                 dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> {
 320                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 321                     for (final VpnToDpnList curDpn : vpnToDpnList) {
 322                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 323                             installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 324                         }
 325                     }
 326                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 327                     futures.add(tx.submit());
 328                     return futures;
 329                 });
 330             }
 331             return;
 332         }
 333
 334         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 335         if (!localDpnIdList.isEmpty()) {
 336             if (vpnToDpnList != null) {
 337                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 338                 dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> {
 339                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 340                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 341                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 342                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 343                                 try {
 344                                     if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 345                                         bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 346                                                 vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx, txnObjects);
 347                                     } else {
 348                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 349                                                 vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 350                                     }
 351                                 } catch (NullPointerException e) {
 352                                     LOG.error("Failed to get create remote fib flows for prefix {} ",
 353                                             vrfEntry.getDestPrefix(), e);
 354                                 }
 355                             }
 356                         }
 357                     }
 358                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 359                     futures.add(tx.submit());
 360                     return futures;
 361                 });
 362             }
 363         }
 364
 365         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 366         if (optVpnUuid.isPresent()) {
 367             String vpnUuid = optVpnUuid.get();
 368             InterVpnLinkDataComposite interVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 369             if (interVpnLink != null) {
 370                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 371                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 372                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 373                         // This is an static route that points to the other endpoint of an InterVpnLink
 374                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 375                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 376                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
 377                     }
 378                 });
 379             }
 380         }
 381     }
 382
 383     void refreshFibTables(String rd, String prefix) {
 384         InstanceIdentifier<VrfEntry> vrfEntryId =
 385                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 386                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 387         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 388         if (vrfEntry.isPresent()) {
 389             createFibEntries(vrfEntryId, vrfEntry.get());
 390         }
 391     }
 392
 393     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 394         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 395         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 396         prefixBuilder.setDpnId(lri.getDpnId());
 397         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 398         prefixBuilder.setIpAddress(lri.getPrefix());
 399         // Increment the refCount here
 400         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 401             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 402         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 403         if (!isPresentInList) {
 404             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 405             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 406             vpnInstanceNames.add(vpnInstanceName);
 407             builder.setVpnInstanceList(vpnInstanceNames);
 408             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 409         } else {
 410             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 411         }
 412         return prefixBuilder.build();
 413     }
 414
 415     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 416                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 417         Boolean wrTxPresent = true;
 418         if (tx == null) {
 419             wrTxPresent = false;
 420             tx = dataBroker.newWriteOnlyTransaction();
 421         }
 422         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 423             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 424             synchronized (label.toString().intern()) {
 425                 LabelRouteInfo lri = getLabelRouteInfo(label);
 426                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 427
 428                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 429                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 430                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 431                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 432                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 433                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 434                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 435                             }
 436                         }
 437                     }
 438                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 439                             label, lri.getVpnInterfaceName(), lri.getDpnId());
 440                 }
 441             }
 442         });
 443         final List<InstructionInfo> instructions = new ArrayList<>();
 444         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(24))
 445             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 446         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 447         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 448         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 449                 NwConstants.ADD_FLOW, tx, null);
 450
 451         if (vrfEntry.getRoutePaths() != null) {
 452             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 453                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 454                     List<ActionInfo> actionsInfos = new ArrayList<>();
 455                     // reinitialize instructions list for LFIB Table
 456                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 457
 458                     actionsInfos.add(new ActionPopMpls());
 459                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 460                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 461                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 462                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 463
 464                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 465                             NwConstants.ADD_FLOW, tx);
 466                 }
 467             }
 468         }
 469         if (!wrTxPresent) {
 470             tx.submit();
 471         }
 472     }
 473
 474     /*
 475      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 476      * LportDispatcher table (via table 80)
 477      */
 478     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 479                                             final VrfEntry vrfEntry) {
 480         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 481         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 482         // packet is commuted from Vpn2 to Vpn1.
 483         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 484         if (!interVpnLink.isActive()) {
 485             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 486                      interVpnLinkName, vrfEntry.getDestPrefix());
 487             return;
 488         }
 489
 490         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 491         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 492         if (!optLportTag.isPresent()) {
 493             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 494             return;
 495         }
 496
 497         Long lportTag = optLportTag.get();
 498         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 499         if (label == null) {
 500             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 501                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 502             return;
 503         }
 504         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 505         List<InstructionInfo> instructions = Arrays.asList(
 506             new InstructionApplyActions(actionsInfos),
 507             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 508                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 509                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 510                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 511             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 512         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 513
 514         for (BigInteger dpId : targetDpns) {
 515             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 516                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 517
 518             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 519                                /*writeTx*/null);
 520         }
 521     }
 522
 523
 524     /*
 525      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 526      */
 527     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 528                                                final VrfEntry vrfEntry, long vpnTag) {
 529         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 530         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 531             && vrfEntry.getRoutePaths().size() == 1);
 532         String destination = vrfEntry.getDestPrefix();
 533         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 534         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 535
 536         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 537         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 538         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 539         if (interVpnLink.getState().or(State.Error) != State.Active) {
 540             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 541                 destination, nextHop, interVpnLinkName);
 542             return;
 543         }
 544
 545         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 546         if (!optOtherEndpointLportTag.isPresent()) {
 547             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 548                 vpnUuid, interVpnLinkName);
 549             return;
 550         }
 551
 552         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 553         if (targetDpns.isEmpty()) {
 554             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 555                 vpnUuid, interVpnLinkName);
 556             return;
 557         }
 558
 559         String[] values = destination.split("/");
 560         String destPrefixIpAddress = values[0];
 561         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 562
 563         List<MatchInfo> matches = new ArrayList<>();
 564         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 565         matches.add(MatchEthernetType.IPV4);
 566
 567         if (prefixLength != 0) {
 568             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 569         }
 570
 571         List<Instruction> instructions =
 572             Arrays.asList(new InstructionWriteMetadata(
 573                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 574                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 575                             .L3VPN_SERVICE_INDEX)),
 576                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 577                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 578
 579         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 580         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 581         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 582             COOKIE_VM_FIB_TABLE, matches, instructions);
 583
 584         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 585             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 586
 587         for (BigInteger dpId : targetDpns) {
 588
 589             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 590                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 591                 dpId, interVpnLink.getInterVpnLinkName());
 592
 593             mdsalManager.installFlow(dpId, flowEntity);
 594         }
 595     }
 596
 597     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 598         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 599         String localNextHopIP = vrfEntry.getDestPrefix();
 600         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, localNextHopIP);
 601         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 602         if (localNextHopInfo == null) {
 603             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 604             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 605                     vpnName, usedRds, localNextHopIP);
 606             boolean localNextHopSeen = false;
 607             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 608             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 609                 String ipPrefix;
 610                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 611                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 612                 } else {
 613                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 614                 }
 615                 Prefixes localNextHopInfoLocal = FibUtil.getPrefixToInterface(dataBroker,
 616                     vpnId, ipPrefix);
 617                 if (localNextHopInfoLocal != null) {
 618                     localNextHopSeen = true;
 619                     BigInteger dpnId =
 620                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 621                                     vpnId, rd, vrfEntry, vpnId, vpnExtraRoute, vpnExtraRoutes);
 622                     returnLocalDpnId.add(dpnId);
 623                 }
 624             }
 625             if (!localNextHopSeen) {
 626                 /* imported routes case */
 627                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 628                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 629                     if (optionalLabel.isPresent()) {
 630                         Long label = optionalLabel.get();
 631                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 632                         synchronized (label.toString().intern()) {
 633                             LabelRouteInfo lri = getLabelRouteInfo(label);
 634                             if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 635                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 636                                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 637                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
 638                                     String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 639                                     if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 640                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 641                                         localNextHopIP = lri.getPrefix();
 642                                     } else {
 643                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 644                                         localNextHopIP = lri.getPrefix();
 645                                     }
 646                                 }
 647                                 if (localNextHopInfo != null) {
 648                                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 649                                             label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 650                                     if (vpnExtraRoutes.isEmpty()) {
 651                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 652                                                 vpnId, rd, vrfEntry, lri.getParentVpnid(), null, vpnExtraRoutes);
 653                                         returnLocalDpnId.add(dpnId);
 654                                     } else {
 655                                         for (Routes extraRoutes : vpnExtraRoutes) {
 656                                             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo,
 657                                                     localNextHopIP,
 658                                                     vpnId, rd, vrfEntry, lri.getParentVpnid(),
 659                                                     extraRoutes, vpnExtraRoutes);
 660                                             returnLocalDpnId.add(dpnId);
 661                                         }
 662                                     }
 663                                 }
 664                             }
 665                         }
 666                     }
 667                 }
 668             }
 669             if (returnLocalDpnId.isEmpty()) {
 670                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 671             }
 672         } else {
 673             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 674                     rd, vrfEntry, vpnId, /*routes*/ null, /*vpnExtraRoutes*/ null);
 675             returnLocalDpnId.add(dpnId);
 676         }
 677         return returnLocalDpnId;
 678     }
 679
 680     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 681                                                 final Long vpnId, final String rd,
 682                                                 final VrfEntry vrfEntry, Long parentVpnId,
 683                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 684         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 685         if (localNextHopInfo != null) {
 686             long groupId;
 687             long localGroupId;
 688             final BigInteger dpnId = localNextHopInfo.getDpnId();
 689             if (Boolean.TRUE.equals(localNextHopInfo.isNatPrefix())) {
 690                 LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip local dpn {} FIB processing",
 691                         vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 692                 return dpnId;
 693             }
 694             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 695             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 696             String prefix = vrfEntry.getDestPrefix();
 697             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 698             //The loadbalancing group is created only if the extra route has multiple nexthops
 699             //to avoid loadbalancing the discovered routes
 700             if (vpnExtraRoutes != null) {
 701                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 702                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 703                 } else {
 704                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 705                 }
 706                 if (vpnExtraRoutes.size() > 1) {
 707                     groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 708                             vpnExtraRoutes);
 709                     localGroupId = nextHopManager.getLocalNextHopGroup(parentVpnId, localNextHopIP);
 710                 } else if (routes.getNexthopIpList().size() > 1) {
 711                     groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 712                             vpnExtraRoutes);
 713                     localGroupId = groupId;
 714                 } else {
 715                     groupId = nextHopManager.getLocalNextHopGroup(parentVpnId, localNextHopIP);
 716                     localGroupId = groupId;
 717                 }
 718             } else {
 719                 groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, interfaceName, localNextHopIP, prefix,
 720                         gwMacAddress, jobKey);
 721                 localGroupId = groupId;
 722             }
 723             if (groupId == FibConstants.INVALID_GROUP_ID) {
 724                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 725                         prefix, rd, interfaceName, dpnId.toString());
 726                 return BigInteger.ZERO;
 727             }
 728             final List<InstructionInfo> instructions = Collections.singletonList(
 729                     new InstructionApplyActions(
 730                             Collections.singletonList(new ActionGroup(groupId))));
 731             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 732                     new InstructionApplyActions(
 733                             Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 734             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 735             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 736             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 737             dataStoreCoordinator.enqueueJob(jobKey, () -> {
 738                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 739                 baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 740                         NwConstants.ADD_FLOW, tx, null);
 741                 if (!FibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(dataBroker,
 742                         localNextHopInfo.getSubnetId(), vpnName, rd)) {
 743                     optLabel.ifPresent(label -> {
 744                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 745                             LOG.debug("Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 746                                             + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 747                                     localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 748                                     nextHopAddressList);
 749                             makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 750                                     NwConstants.ADD_FLOW, tx);
 751                             // If the extra-route is reachable from VMs attached to the same switch,
 752                             // then the tunnel table can point to the load balancing group.
 753                             // If it is reachable from VMs attached to different switches,
 754                             // then it should be pointing to one of the local group in order to avoid looping.
 755                             if (vrfEntry.getRoutePaths().size() == 1) {
 756                                 makeTunnelTableEntry(dpnId, label, groupId, tx);
 757                             } else {
 758                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 759                             }
 760                         } else {
 761                             LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 762                                             + "route. LFib and Terminating table entries will not be created.",
 763                                     rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 764                         }
 765                     });
 766                 }
 767                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 768                 futures.add(tx.submit());
 769                 return futures;
 770             });
 771             return dpnId;
 772         }
 773         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 774                 vpnName);
 775         return BigInteger.ZERO;
 776     }
 777
 778     private LabelRouteInfo getLabelRouteInfo(Long label) {
 779         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 780             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 781         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 782         if (opResult.isPresent()) {
 783             return opResult.get();
 784         }
 785         return null;
 786     }
 787
 788     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 789         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 790         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 791             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 792         if (lri == null) {
 793             return true;
 794         }
 795         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 796             ? lri.getVpnInstanceList() : new ArrayList<>();
 797         if (vpnInstancesList.contains(vpnInstanceName)) {
 798             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 799             vpnInstancesList.remove(vpnInstanceName);
 800         }
 801         if (vpnInstancesList.size() == 0) {
 802             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 803             if (tx != null) {
 804                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 805             } else {
 806                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 807             }
 808             return true;
 809         } else {
 810             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 811             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 812             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 813         }
 814         return false;
 815     }
 816
 817     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 818                                       WriteTransaction tx) {
 819         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 820
 821         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 822
 823         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 824             dpId, label, groupId);
 825     }
 826
 827     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 828                                                 WriteTransaction tx) {
 829         List<MatchInfo> mkMatches = new ArrayList<>();
 830
 831         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 832             destDpId, label, actionsInfos);
 833
 834         // Matching metadata
 835         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 836         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 837
 838         List<InstructionInfo> mkInstructions = new ArrayList<>();
 839         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 840
 841         FlowEntity terminatingServiceTableFlowEntity =
 842             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 843             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 844                 String.format("%s:%d", "TST Flow Entry ", label),
 845             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 846
 847         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 848
 849         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 850
 851         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 852         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 853             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 854             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 855             .child(Flow.class, flowKey).build();
 856         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
 857                 WriteTransaction.CREATE_MISSING_PARENTS);
 858     }
 859
 860     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 861         FlowEntity flowEntity;
 862         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
 863         List<MatchInfo> mkMatches = new ArrayList<>();
 864         // Matching metadata
 865         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 866         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 867             NwConstants.INTERNAL_TUNNEL_TABLE,
 868             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
 869             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
 870             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 871         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
 872         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 873         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 874             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 875             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 876
 877         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 878         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 879     }
 880
 881     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 882         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 883         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 884         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 885         boolean isExtraroute = false;
 886         if (localNextHopInfo == null) {
 887             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
 888             if (usedRds.size() > 1) {
 889                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
 890                         vrfEntry.getDestPrefix(), vpnName, rd);
 891                 return returnLocalDpnId;
 892             }
 893             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
 894             //in the vpn
 895             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
 896                     vpnName, rd, vrfEntry.getDestPrefix());
 897             if (extraRouteOptional.isPresent()) {
 898                 isExtraroute = true;
 899                 Routes extraRoute = extraRouteOptional.get();
 900                 String ipPrefix;
 901                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
 902                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 903                 } else {
 904                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 905                 }
 906                 localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
 907                 if (localNextHopInfo != null) {
 908                     String localNextHopIP = localNextHopInfo.getIpAddress();
 909                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 910                             vpnId, rd, vrfEntry, isExtraroute);
 911                     if (!dpnId.equals(BigInteger.ZERO)) {
 912                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
 913                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
 914                                 /*remove*/ false);
 915                         returnLocalDpnId.add(dpnId);
 916                     }
 917                 } else {
 918                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
 919                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
 920                 }
 921             }
 922
 923             if (localNextHopInfo == null) {
 924                 /* Imported VRF entry */
 925                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 926                 if (optionalLabel.isPresent()) {
 927                     Long label = optionalLabel.get();
 928                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 929                     LabelRouteInfo lri = getLabelRouteInfo(label);
 930                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 931                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 932                         prefixBuilder.setDpnId(lri.getDpnId());
 933                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
 934                                 vpnId, rd, vrfEntry, isExtraroute);
 935                         if (!dpnId.equals(BigInteger.ZERO)) {
 936                             returnLocalDpnId.add(dpnId);
 937                         }
 938                     }
 939                 }
 940             }
 941
 942         } else {
 943             String localNextHopIP = localNextHopInfo.getIpAddress();
 944             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 945                 vpnId, rd, vrfEntry, isExtraroute);
 946             if (!dpnId.equals(BigInteger.ZERO)) {
 947                 returnLocalDpnId.add(dpnId);
 948             }
 949         }
 950
 951         return returnLocalDpnId;
 952     }
 953
 954     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
 955                                                 final Long vpnId, final String rd,
 956                                                 final VrfEntry vrfEntry, boolean isExtraroute) {
 957         if (localNextHopInfo != null) {
 958             final BigInteger dpnId = localNextHopInfo.getDpnId();
 959             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 960             dataStoreCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId,
 961                     vrfEntry.getDestPrefix()), () -> {
 962                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 963                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
 964                             NwConstants.DEL_FLOW, tx, null);
 965                     if (!FibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(dataBroker,
 966                             localNextHopInfo.getSubnetId(), vpnId, rd)) {
 967                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 968                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 969                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
 970                                         NwConstants.DEL_FLOW, tx);
 971                                 removeTunnelTableEntry(dpnId, label, tx);
 972                             });
 973                         }
 974                     }
 975                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 976                     futures.add(tx.submit());
 977                     return futures;
 978                 });
 979             //TODO: verify below adjacency call need to be optimized (?)
 980             //In case of the removal of the extra route, the loadbalancing group is updated
 981             if (!isExtraroute) {
 982                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
 983             }
 984             return dpnId;
 985         }
 986         return BigInteger.ZERO;
 987     }
 988
 989     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
 990             final VrfEntry vrfEntry, WriteTransaction tx) {
 991         Boolean wrTxPresent = true;
 992         if (tx == null) {
 993             wrTxPresent = false;
 994             tx = dataBroker.newWriteOnlyTransaction();
 995         }
 996
 997         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 998         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
 999                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1000
1001         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1002         if (adjacencyResults == null || adjacencyResults.isEmpty()) {
1003             LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
1004             LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1005             return;
1006         }
1007
1008         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1009         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1010                 vpnName, usedRds, vrfEntry.getDestPrefix());
1011         // create loadbalancing groups for extra routes only when the extra route is present behind
1012         // multiple VMs
1013         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1014                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1015             List<InstructionInfo> instructions = new ArrayList<>();
1016             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1017                     null, vpnExtraRoutes);
1018             if (groupId == FibConstants.INVALID_GROUP_ID) {
1019                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1020                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1021                 return;
1022             }
1023             List<ActionInfo> actionInfos =
1024                     Collections.singletonList(new ActionGroup(groupId));
1025             instructions.add(new InstructionApplyActions(actionInfos));
1026             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1027                     NwConstants.ADD_FLOW, tx, null);
1028         } else {
1029             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1030         }
1031
1032         if (!wrTxPresent) {
1033             tx.submit();
1034         }
1035         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1036     }
1037
1038     void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1039     /* Get interface info from prefix to interface mapping;
1040         Use the interface info to get the corresponding vpn interface op DS entry,
1041         remove the adjacency corresponding to this fib entry.
1042         If adjacency removed is the last adjacency, clean up the following:
1043          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1044          - prefix to interface entry
1045          - vpn interface op DS
1046      */
1047         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1048         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1049         Routes extraRoute = null;
1050         if (prefixInfo == null) {
1051             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1052             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1053             extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1054             if (extraRoute != null) {
1055                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1056                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1057                     if (nextHopIp != null) {
1058                         String ipPrefix;
1059                         if (isIpv4Address(nextHopIp)) {
1060                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1061                         } else {
1062                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1063                         }
1064                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1065                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1066                     }
1067                 }
1068             }
1069             if (prefixInfo == null) {
1070                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1071                 if (optionalLabel.isPresent()) {
1072                     Long label = optionalLabel.get();
1073                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1074                     LabelRouteInfo lri = getLabelRouteInfo(label);
1075                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1076                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1077                         prefixBuilder.setDpnId(lri.getDpnId());
1078                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1079                         prefixBuilder.setIpAddress(lri.getPrefix());
1080                         prefixInfo = prefixBuilder.build();
1081                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1082                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1083                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1084                     }
1085                 }
1086             }
1087         } else {
1088             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1089         }
1090     }
1091
1092     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1093                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1094
1095         if (prefixInfo == null) {
1096             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1097             return; //Don't have any info for this prefix (shouldn't happen); need to return
1098         }
1099
1100         if (Boolean.TRUE.equals(prefixInfo.isNatPrefix())) {
1101             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1102                     vrfEntry.getDestPrefix(), vpnId, rd);
1103             return;
1104         }
1105
1106         String ifName = prefixInfo.getVpnInterfaceName();
1107         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1108         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1109             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1110     }
1111
1112     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1113         Prefixes prefixInfo;
1114         Long vpnId;
1115         String rd;
1116         VrfEntry vrfEntry;
1117         Routes extraRoute;
1118
1119         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1120                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1121             this.prefixInfo = prefixInfo;
1122             this.vpnId = vpnId;
1123             this.rd = rd;
1124             this.vrfEntry = vrfEntry;
1125             this.extraRoute = extraRoute;
1126         }
1127
1128         @Override
1129         public List<ListenableFuture<Void>> call() throws Exception {
1130             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1131             // to call the respective helpers.
1132             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1133
1134             //First Cleanup LabelRouteInfo
1135             //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1136             if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Mplsgre)) {
1137                 FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1138                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1139                     synchronized (label.toString().intern()) {
1140                         LabelRouteInfo lri = getLabelRouteInfo(label);
1141                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1142                                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1143                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1144                                     FibUtil.getVpnInstanceOpData(dataBroker, rd);
1145                             String vpnInstanceName = "";
1146                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
1147                                 vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1148                             }
1149                             boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, writeOperTxn);
1150                             if (lriRemoved) {
1151                                 String parentRd = lri.getParentVpnRd();
1152                                 FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1153                                         FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1154                             }
1155                         } else {
1156                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1157                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1158                         }
1159                     }
1160                 });
1161             }
1162             String ifName = prefixInfo.getVpnInterfaceName();
1163             Optional<VpnInterface> optvpnInterface = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1164                 FibUtil.getVpnInterfaceIdentifier(ifName));
1165             if (optvpnInterface.isPresent()) {
1166                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1167                 if (vpnId != associatedVpnId) {
1168                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1169                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1170                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1171                     return null;
1172                 } else {
1173                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1174                         vrfEntry.getDestPrefix(), associatedVpnId);
1175                 }
1176             }
1177             if (extraRoute != null) {
1178                 Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, rd);
1179                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1180                 //Only one used Rd present in case of removal event
1181                 String usedRd = usedRds.get(0);
1182                 if (optVpnName.isPresent()) {
1183                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1184                             baseVrfEntryHandler.getVpnToExtrarouteIdentifier(optVpnName.get(), usedRd,
1185                                     vrfEntry.getDestPrefix()));
1186                     writeOperTxn.delete(LogicalDatastoreType.CONFIGURATION,
1187                             VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1188                 }
1189             }
1190             Optional<Adjacencies> optAdjacencies = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1191                 FibUtil.getAdjListPath(ifName));
1192             int numAdj = 0;
1193             if (optAdjacencies.isPresent()) {
1194                 numAdj = optAdjacencies.get().getAdjacency().size();
1195             }
1196             //remove adjacency corr to prefix
1197             if (numAdj > 1) {
1198                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId,
1199                         vrfEntry.getDestPrefix());
1200                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1201                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1202             } else {
1203                 //this is last adjacency (or) no more adjacency left for this vpn interface, so
1204                 //clean up the vpn interface from DpnToVpn list
1205                 LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1206                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName));
1207             }
1208             List<ListenableFuture<Void>> futures = new ArrayList<>();
1209             futures.add(writeOperTxn.submit());
1210             return futures;
1211         }
1212     }
1213
1214     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1215         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1216         final String rd = vrfTableKey.getRouteDistinguisher();
1217         final VpnInstanceOpDataEntry vpnInstance =
1218                 FibUtil.getVpnInstance(dataBroker, vrfTableKey.getRouteDistinguisher());
1219         if (vpnInstance == null) {
1220             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1221             return;
1222         }
1223         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1224         long elanTag = 0L;
1225         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1226         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1227         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1228         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1229         if (subnetRoute != null) {
1230             elanTag = subnetRoute.getElantag();
1231             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1232                 rd, vrfEntry.getDestPrefix(), elanTag);
1233             if (vpnToDpnList != null) {
1234                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1235                 dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1236                     () -> {
1237                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1238
1239                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1240
1241                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1242                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1243                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1244                                 optionalLabel.ifPresent(label -> {
1245                                     makeLFibTableEntry(curDpn.getDpnId(), label, null,
1246                                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1247                                 });
1248                             }
1249                         }
1250                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1251                         futures.add(tx.submit());
1252                         return futures;
1253                     });
1254             }
1255             optionalLabel.ifPresent(label -> {
1256                 synchronized (label.toString().intern()) {
1257                     LabelRouteInfo lri = getLabelRouteInfo(label);
1258                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1259                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1260                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
1261                         String vpnInstanceName = "";
1262                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1263                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1264                         }
1265                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1266                         if (lriRemoved) {
1267                             String parentRd = lri.getParentVpnRd();
1268                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1269                                     FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1270                             LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1271                                     + "labelRouteInfo cleared", label, rd,
1272                                     vrfEntry.getDestPrefix());
1273                         }
1274                     } else {
1275                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1276                                 FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1277                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1278                                 label, rd, vrfEntry.getDestPrefix());
1279                     }
1280                 }
1281             });
1282             return;
1283         }
1284
1285         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1286             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1287         if (vpnToDpnList != null) {
1288             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1289                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1290             String jobKey;
1291             Optional<Routes> extraRouteOptional;
1292             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1293             if (usedRds != null && !usedRds.isEmpty()) {
1294                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1295                 if (usedRds.size() > 1) {
1296                     LOG.error("The extra route prefix is still present in some DPNs");
1297                     return ;
1298                 } else {
1299                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1300                     //is not present in any other DPN
1301                     extraRouteOptional = VpnExtraRouteHelper
1302                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1303                 }
1304             } else {
1305                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1306                 extraRouteOptional = Optional.absent();
1307             }
1308             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1309             dataStoreCoordinator.enqueueJob(jobKey,
1310                 () -> {
1311                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1312
1313                     if (localDpnIdList.size() <= 0) {
1314                         for (VpnToDpnList curDpn : vpnToDpnList) {
1315                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1316                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1317                         }
1318                     } else {
1319                         for (BigInteger localDpnId : localDpnIdList) {
1320                             for (VpnToDpnList curDpn : vpnToDpnList) {
1321                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1322                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1323                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1324                                 }
1325                             }
1326                         }
1327                     }
1328                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1329                     futures.add(tx.submit());
1330                     return futures;
1331                 });
1332         }
1333
1334         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1335         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1336         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1337
1338         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1339         // of the interVpnLink.
1340         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1341         if (optVpnUuid.isPresent()) {
1342             String vpnUuid = optVpnUuid.get();
1343             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1344                 Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1345                 if (optInterVpnLink.isPresent()) {
1346                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1347                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1348                         // This is route that points to the other endpoint of an InterVpnLink
1349                         // In that case, we should look for the FIB table pointing to
1350                         // LPortDispatcher table and remove it.
1351                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1352                     }
1353                 }
1354             });
1355         }
1356
1357     }
1358
1359     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1360                                     int addOrRemove, WriteTransaction tx) {
1361         Boolean wrTxPresent = true;
1362         if (tx == null) {
1363             wrTxPresent = false;
1364             tx = dataBroker.newWriteOnlyTransaction();
1365         }
1366
1367         List<MatchInfo> matches = new ArrayList<>();
1368         matches.add(MatchEthernetType.MPLS_UNICAST);
1369         matches.add(new MatchMplsLabel(label));
1370
1371         // Install the flow entry in L3_LFIB_TABLE
1372         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1373
1374         FlowEntity flowEntity;
1375         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1376             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1377         Flow flow = flowEntity.getFlowBuilder().build();
1378         String flowId = flowEntity.getFlowId();
1379         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1380         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1381         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1382             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1383             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1384
1385         if (addOrRemove == NwConstants.ADD_FLOW) {
1386             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1387         } else {
1388             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1389         }
1390         if (!wrTxPresent) {
1391             tx.submit();
1392         }
1393
1394         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1395             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1396     }
1397
1398     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1399                                     final FutureCallback<List<Void>> callback) {
1400         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1401         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1402         final VpnInstanceOpDataEntry vpnInstance = FibUtil.getVpnInstance(dataBroker, rd);
1403         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1404         List<SubTransaction> txnObjects =  new ArrayList<>();
1405         if (!vrfTable.isPresent()) {
1406             LOG.warn("VRF Table not yet available for RD {}", rd);
1407             if (callback != null) {
1408                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1409                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1410                 Futures.addCallback(listenableFuture, callback);
1411             }
1412             return;
1413         }
1414         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1415         dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1416             () -> {
1417                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1418                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1419                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1420                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1421                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1422                         if (subnetRoute != null) {
1423                             long elanTag = subnetRoute.getElantag();
1424                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1425                             continue;
1426                         }
1427                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1428                         if (routerInt != null) {
1429                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1430                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1431                             routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1432                                     routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1433                                     NwConstants.ADD_FLOW);
1434                             continue;
1435                         }
1436                         //Handle local flow creation for imports
1437                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1438                             java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1439                             if (optionalLabel.isPresent()) {
1440                                 List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1441                                 LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1442                                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1443                                     if (lri.getDpnId().equals(dpnId)) {
1444                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1445                                         continue;
1446                                     }
1447                                 }
1448                             }
1449                         }
1450
1451                         boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1452                                 vrfEntry, dpnId);
1453                         if (shouldCreateRemoteFibEntry) {
1454                             LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}",
1455                                     vrfEntry, dpnId);
1456                             if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1457                                 bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1458                                         vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1459                             } else {
1460                                 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1461                                         vrfEntry, tx);
1462                             }
1463                         }
1464                     }
1465                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1466                     futures.add(tx.submit());
1467                 }
1468                 if (callback != null) {
1469                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1470                     Futures.addCallback(listenableFuture, callback);
1471                 }
1472                 return futures;
1473             });
1474     }
1475
1476     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1477                                             final String localNextHopIp, final String remoteNextHopIp) {
1478         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1479             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1480         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1481         final VpnInstanceOpDataEntry vpnInstance = FibUtil.getVpnInstance(dataBroker, rd);
1482         List<SubTransaction> txnObjects =  new ArrayList<>();
1483         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1484         if (vrfTable.isPresent()) {
1485             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1486             dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1487                 () -> {
1488                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1489                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1490                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1491                         vrfTable.get().getVrfEntry().stream()
1492                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1493                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1494                                         rd, remoteNextHopIp, vrfTable, writeCfgTxn, txnObjects));
1495                         futures.add(writeCfgTxn.submit());
1496                     }
1497                     return futures;
1498                 });
1499         }
1500     }
1501
1502     public void manageRemoteRouteOnDPN(final boolean action,
1503                                        final BigInteger localDpnId,
1504                                        final long vpnId,
1505                                        final String rd,
1506                                        final String destPrefix,
1507                                        final String destTepIp,
1508                                        final long label) {
1509         final VpnInstanceOpDataEntry vpnInstance = FibUtil.getVpnInstance(dataBroker, rd);
1510
1511         if (vpnInstance == null) {
1512             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1513             return;
1514         }
1515         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1516         dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1517             () -> {
1518                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1519                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1520                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1521                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1522                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1523                     if (vrfEntry == null) {
1524                         return futures;
1525                     }
1526                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1527                         action, localDpnId, vpnId, rd, destPrefix);
1528                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1529                     VrfEntry modVrfEntry;
1530                     if (routePathList == null || (routePathList.isEmpty())) {
1531                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1532                                 Collections.singletonList(destTepIp),
1533                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1534                     } else {
1535                         modVrfEntry = vrfEntry;
1536                     }
1537
1538                     if (action == true) {
1539                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1540                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1541                                 modVrfEntry, writeTransaction);
1542                     } else {
1543                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1544                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1545                                 vrfEntry.getDestPrefix());
1546                         if (usedRds.size() > 1) {
1547                             LOG.debug("The extra route prefix is still present in some DPNs");
1548                             return futures;
1549                         }
1550                         //Is this fib route an extra route? If yes, get the nexthop which would be
1551                         //an adjacency in the vpn
1552                         Optional<Routes> extraRouteOptional = Optional.absent();
1553                         if (usedRds.size() != 0) {
1554                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1555                                     FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId()),
1556                                     usedRds.get(0), vrfEntry.getDestPrefix());
1557                         }
1558                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1559                                 extraRouteOptional, writeTransaction);
1560                     }
1561                     futures.add(writeTransaction.submit());
1562                 }
1563                 return futures;
1564             });
1565     }
1566
1567     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1568                                  final FutureCallback<List<Void>> callback) {
1569         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1570         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1571         final VpnInstanceOpDataEntry vpnInstance = FibUtil.getVpnInstance(dataBroker, rd);
1572         List<SubTransaction> txnObjects =  new ArrayList<>();
1573         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1574         if (vrfTable.isPresent()) {
1575             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1576             dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1577                 () -> {
1578                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1579                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1580                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1581                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1582                                 /* Handle subnet routes here */
1583                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1584                             if (subnetRoute != null) {
1585                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
1586                                         vrfEntry.getDestPrefix(),
1587                                         dpnId, rd);
1588                                 baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1589                                         NwConstants.DEL_FLOW, tx, null);
1590                                 List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1591                                 if (routePaths != null) {
1592                                     for (RoutePaths routePath : routePaths) {
1593                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1594                                                 DEFAULT_FIB_FLOW_PRIORITY,
1595                                                 NwConstants.DEL_FLOW, tx);
1596                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} "
1597                                                         + "for rd {} prefix {}",
1598                                                 routePath.getLabel(), rd,
1599                                                 vrfEntry.getDestPrefix());
1600                                     }
1601                                 }
1602                                 continue;
1603                             }
1604                             // ping responder for router interfaces
1605                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1606                             if (routerInt != null) {
1607                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1608                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1609                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1610                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1611                                         NwConstants.DEL_FLOW);
1612                                 continue;
1613                             }
1614                             // Passing null as we don't know the dpn
1615                             // to which prefix is attached at this point
1616                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1617                                     vrfEntry.getDestPrefix());
1618                             String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1619                             Optional<Routes> extraRouteOptional;
1620                             //Is this fib route an extra route? If yes, get the nexthop which would be
1621                             //an adjacency in the vpn
1622                             if (usedRds != null && !usedRds.isEmpty()) {
1623                                 if (usedRds.size() > 1) {
1624                                     LOG.error("The extra route prefix is still present in some DPNs");
1625                                     return futures;
1626                                 } else {
1627                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1628                                             usedRds.get(0), vrfEntry.getDestPrefix());
1629
1630                                 }
1631                             } else {
1632                                 extraRouteOptional = Optional.absent();
1633                             }
1634                             if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1635                                 bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
1636                                         vrfEntry, extraRouteOptional, tx, txnObjects);
1637                             } else {
1638                                 baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
1639                                         vrfEntry, extraRouteOptional, tx);
1640                             }
1641                         }
1642                         futures.add(tx.submit());
1643                         if (callback != null) {
1644                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1645                             Futures.addCallback(listenableFuture, callback);
1646                         }
1647                     }
1648                     return futures;
1649                 });
1650         }
1651     }
1652
1653     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1654                                            final String localNextHopIp, final String remoteNextHopIp) {
1655         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1656                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1657             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1658         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1659         final VpnInstanceOpDataEntry vpnInstance = FibUtil.getVpnInstance(dataBroker, rd);
1660         List<SubTransaction> txnObjects =  new ArrayList<>();
1661         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1662         if (vrfTable.isPresent()) {
1663             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1664             dataStoreCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1665                 () -> {
1666                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1667                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1668                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1669                         vrfTable.get().getVrfEntry().stream()
1670                             .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1671                             .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
1672                                 remoteNextHopIp, vrfTable, writeCfgTxn, txnObjects));
1673                         futures.add(writeCfgTxn.submit());
1674                     }
1675                     return futures;
1676                 });
1677         }
1678     }
1679
1680     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1681         InstanceIdentifierBuilder<VrfTables> idBuilder =
1682             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1683         InstanceIdentifier<VrfTables> id = idBuilder.build();
1684         return id;
1685     }
1686
1687     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1688         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1689                 .FLOWID_SEPARATOR + nextHop;
1690     }
1691
1692     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1693         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1694                 + tableMiss + FLOWID_PREFIX;
1695     }
1696
1697     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1698         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1699             .child(VrfTables.class, new VrfTablesKey(rd))
1700             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1701         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1702         if (vrfEntry.isPresent()) {
1703             return vrfEntry.get();
1704         }
1705         return null;
1706     }
1707
1708     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1709                                              final String vpnName,
1710                                              final VrfEntry vrfEntry) {
1711         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1712
1713         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1714         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1715
1716         if (targetDpns.isEmpty()) {
1717             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1718             return;
1719         }
1720
1721         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1722         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1723
1724         // delete from FIB
1725         //
1726         optNextHop.ifPresent(nextHop -> {
1727             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1728             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1729             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
1730                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1731
1732             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1733             for (BigInteger dpId : targetDpns) {
1734                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1735                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1736
1737                 mdsalManager.removeFlow(dpId, flow);
1738             }
1739         });
1740
1741         // delete from LFIB
1742         //
1743         optLabel.ifPresent(label -> {
1744             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1745
1746             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1747             for (BigInteger dpId : targetDpns) {
1748                 LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1749                           vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1750                 makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
1751             }
1752             tx.submit();
1753         });
1754     }
1755
1756     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1757             List<String> nextHopAddressList, LabelRouteInfo lri) {
1758         return lri != null && lri.getPrefix().equals(prefix)
1759                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1760     }
1761
1762     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1763         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1764             return true;
1765         }
1766
1767         Prefixes prefix = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1768         if (prefix != null) {
1769             BigInteger prefixDpnId = prefix.getDpnId();
1770             if (prefixDpnId == dpnId) {
1771                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1772                         vrfEntry, dpnId);
1773                 return false;
1774             }
1775         }
1776
1777         return true;
1778     }
1779 }