fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  42 import org.opendaylight.genius.mdsalutil.ActionInfo;
  43 import org.opendaylight.genius.mdsalutil.FlowEntity;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  48 import org.opendaylight.genius.mdsalutil.NWUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.UpgradeState;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  54 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  57 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  63 import org.opendaylight.genius.utils.ServiceIndex;
  64 import org.opendaylight.genius.utils.batching.SubTransaction;
  65 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  66 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  67 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  68 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  69 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  70 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  71 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  72 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  73 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  74 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 110 import org.slf4j.Logger;
 111 import org.slf4j.LoggerFactory;
 112
 113
 114 @Singleton
 115 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 118     private static final String FLOWID_PREFIX = "L3.";
 119     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 120     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 121     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 122     private static final int LFIB_INTERVPN_PRIORITY = 15;
 123     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 124     private static final int MAX_RETRIES = 3;
 125     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 126
 127     private final DataBroker dataBroker;
 128     private final ManagedNewTransactionRunner txRunner;
 129     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 130     private final IMdsalApiManager mdsalManager;
 131     private final NexthopManager nextHopManager;
 132     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 133     private final BaseVrfEntryHandler baseVrfEntryHandler;
 134     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 135     private final JobCoordinator jobCoordinator;
 136     private final IElanService elanManager;
 137     private final FibUtil fibUtil;
 138     private final InterVpnLinkCache interVpnLinkCache;
 139     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 140     private final UpgradeState upgradeState;
 141     private final DataTreeEventCallbackRegistrar eventCallbacks;
 142
 143     @Inject
 144     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 145                             final NexthopManager nexthopManager,
 146                             final IElanService elanManager,
 147                             final BaseVrfEntryHandler vrfEntryHandler,
 148                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 149                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 150                             final JobCoordinator jobCoordinator,
 151                             final FibUtil fibUtil,
 152                             final InterVpnLinkCache interVpnLinkCache,
 153                             final UpgradeState upgradeState,
 154                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 155         super(VrfEntry.class, VrfEntryListener.class);
 156         this.dataBroker = dataBroker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 158         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 159         this.mdsalManager = mdsalApiManager;
 160         this.nextHopManager = nexthopManager;
 161         this.elanManager = elanManager;
 162         this.baseVrfEntryHandler = vrfEntryHandler;
 163         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 164         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.fibUtil = fibUtil;
 167         this.interVpnLinkCache = interVpnLinkCache;
 168         this.upgradeState = upgradeState;
 169         this.eventCallbacks = eventCallbacks;
 170     }
 171
 172     @Override
 173     @PostConstruct
 174     public void init() {
 175         LOG.info("{} init", getClass().getSimpleName());
 176         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 177     }
 178
 179     @Override
 180     @SuppressWarnings("checkstyle:IllegalCatch")
 181     public void close() {
 182         closeables.forEach(c -> {
 183             try {
 184                 c.close();
 185             } catch (Exception e) {
 186                 LOG.warn("Error closing {}", c, e);
 187             }
 188         });
 189     }
 190
 191     @Override
 192     protected VrfEntryListener getDataTreeChangeListener() {
 193         return VrfEntryListener.this;
 194     }
 195
 196     @Override
 197     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 198         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 199     }
 200
 201     @Override
 202     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 203         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 204         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 205         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 206                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 207         addFibEntries(identifier, vrfEntry, rd);
 208         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 209                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 210     }
 211
 212     //This method is temporary. Eventually Factory design pattern will be used to get
 213     // right VrfEntryhandle and invoke its methods.
 214     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 215         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 216             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 217             return;
 218         }
 219         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 220             LOG.info("EVPN flows need to be programmed.");
 221             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 222                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 223             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 224             closeables.add(evpnVrfEntryHandler);
 225             return;
 226         }
 227         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 228         if (routerInt != null) {
 229             // ping responder for router interfaces
 230             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 234             createFibEntries(identifier, vrfEntry);
 235             return;
 236         }
 237     }
 238
 239     @Override
 240     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 241         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 242         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 243         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 244                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 245         removeFibEntries(identifier, vrfEntry, rd);
 246         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 247             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 248     }
 249
 250     //This method is temporary. Eventually Factory design pattern will be used to get
 251     // right VrfEntryhandle and invoke its methods.
 252     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 253         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 254             LOG.info("EVPN flows to be deleted");
 255             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 256                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 257             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 258             closeables.add(evpnVrfEntryHandler);
 259             return;
 260         }
 261         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 262         if (routerInt != null) {
 263             // ping responder for router interfaces
 264             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 265             return;
 266         }
 267         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 268             deleteFibEntries(identifier, vrfEntry);
 269             return;
 270         }
 271         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 272             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 273             return;
 274         }
 275     }
 276
 277     @Override
 278     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 279     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 280     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 281     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 282         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 283         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 284         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 285                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 286         // Handle BGP Routes first
 287         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 288             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 289             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 290                     rd, update.getDestPrefix(), update.getRoutePaths());
 291             return;
 292         }
 293
 294         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 295             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 296             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 297             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 298
 299             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 300             //updated vrf entry route path is null. This can happen during tunnel events.
 301             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 302             List<String> usedRds = new ArrayList<>();
 303             if (optVpnInstance.isPresent()) {
 304                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 305                         update.getDestPrefix());
 306             }
 307             // If original VRF Entry had nexthop null , but update VRF Entry
 308             // has nexthop , route needs to be created on remote Dpns
 309             if (originalRoutePath == null || originalRoutePath.isEmpty()
 310                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 311                 // TODO(vivek): Though ugly, Not handling this code now, as each
 312                 // tep add event will invoke flow addition
 313                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 314                         + " This event is IGNORED here.", update.getDestPrefix());
 315                 return;
 316             }
 317
 318             // If original VRF Entry had valid nexthop , but update VRF Entry
 319             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 320             if (updateRoutePath == null || updateRoutePath.isEmpty()
 321                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 322                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 323                         + "This event is IGNORED here.", update.getDestPrefix());
 324                 return;
 325             }
 326             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 327             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 328             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 329             List<ListenableFuture<Void>> futures = new ArrayList<>();
 330             ListenableFuture<Void> configFuture =
 331                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(configTx ->
 332                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
 333                                     nextHopsRemoved.parallelStream()
 334                                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 335                                                     configTx, operTx, nextHopRemoved, rd, update.getDestPrefix())))));
 336             futures.add(configFuture);
 337             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 338                 @Override
 339                 public void onSuccess(Void result) {
 340                     createFibEntries(identifier, update);
 341                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 342                             rd, update.getDestPrefix(), update.getRoutePaths());
 343                 }
 344
 345                 @Override
 346                 public void onFailure(Throwable throwable) {
 347                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 348                             update, throwable);
 349                 }
 350             }, MoreExecutors.directExecutor());
 351             return;
 352         }
 353
 354         //Handle all other routes only on a cluster reboot
 355         if (original.equals(update)) {
 356             //Reboot use-case
 357             createFibEntries(identifier, update);
 358             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 359                     rd, update.getDestPrefix(), update.getRoutePaths());
 360             return;
 361         }
 362
 363         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 364                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 365     }
 366
 367     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 368         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 369         List<SubTransaction> txnObjects =  new ArrayList<>();
 370         final VpnInstanceOpDataEntry vpnInstance =
 371                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 372         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 373         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 374                 + " has null vpnId!");
 375         final Collection<VpnToDpnList> vpnToDpnList;
 376         if (vrfEntry.getParentVpnRd() != null
 377                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 378             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 379             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 380             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 381                 vpnInstance.getVpnToDpnList();
 382             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 383                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 384         } else {
 385             vpnToDpnList = vpnInstance.getVpnToDpnList();
 386         }
 387         final Long vpnId = vpnInstance.getVpnId();
 388         final String rd = vrfTableKey.getRouteDistinguisher();
 389         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 390         if (subnetRoute != null) {
 391             final long elanTag = subnetRoute.getElantag();
 392             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 393                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 394             if (vpnToDpnList != null) {
 395                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 396                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 397                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 398                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 399                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 400                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 401                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 402                             }
 403                         }
 404                     })));
 405             }
 406             return;
 407         }
 408         // Get etherType value based on the IpPrefix address family type
 409         int etherType;
 410         try {
 411             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 412         } catch (IllegalArgumentException ex) {
 413             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 414             return;
 415         }
 416
 417         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 418         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 419             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 420                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 421                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 422                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 423                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 424                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 425                                     try {
 426                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 427                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 428                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 429                                                     txnObjects);
 430                                         } else {
 431                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 432                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 433                                         }
 434                                     } catch (NullPointerException e) {
 435                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 436                                                 vrfEntry.getDestPrefix(), e);
 437                                     }
 438                                 }
 439                             }
 440                         }
 441                     }
 442                 })), MAX_RETRIES);
 443         }
 444
 445         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 446         if (optVpnUuid.isPresent()) {
 447             String vpnUuid = optVpnUuid.get();
 448             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 449             if (interVpnLink != null) {
 450                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 451                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 452                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 453                         // This is an static route that points to the other endpoint of an InterVpnLink
 454                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 455                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 456                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 457                     }
 458                 });
 459             }
 460         }
 461     }
 462
 463     void refreshFibTables(String rd, String prefix) {
 464         InstanceIdentifier<VrfEntry> vrfEntryId =
 465                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 466                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 467         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 468         if (vrfEntry.isPresent()) {
 469             createFibEntries(vrfEntryId, vrfEntry.get());
 470         }
 471     }
 472
 473     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 474         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 475         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 476         prefixBuilder.setDpnId(lri.getDpnId());
 477         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 478         prefixBuilder.setIpAddress(lri.getPrefix());
 479         // Increment the refCount here
 480         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 481             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 482         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 483         if (!isPresentInList) {
 484             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 485             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 486             vpnInstanceNames.add(vpnInstanceName);
 487             builder.setVpnInstanceList(vpnInstanceNames);
 488             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 489         } else {
 490             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 491         }
 492         return prefixBuilder.build();
 493     }
 494
 495     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 496                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 497         if (tx == null) {
 498             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 499                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 500                 "Error installing subnet route in FIB");
 501             return;
 502         }
 503         int etherType;
 504         try {
 505             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 506         } catch (IllegalArgumentException ex) {
 507             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 508             return;
 509         }
 510         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 511             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 512             synchronized (label.toString().intern()) {
 513                 LabelRouteInfo lri = getLabelRouteInfo(label);
 514                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 515
 516                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 517                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 518                                 fibUtil.getVpnInstanceOpData(rd);
 519                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 520                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 521                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 522                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 523                             }
 524                         }
 525                     }
 526                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 527                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 528                 }
 529             }
 530         });
 531         final List<InstructionInfo> instructions = new ArrayList<>();
 532         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 533             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 534         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 535         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 536         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 537                 NwConstants.ADD_FLOW, tx, null);
 538         if (vrfEntry.getRoutePaths() != null) {
 539             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 540                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 541                     List<ActionInfo> actionsInfos = new ArrayList<>();
 542                     // reinitialize instructions list for LFIB Table
 543                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 544                     actionsInfos.add(new ActionPopMpls(etherType));
 545                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 546                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 547                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 548                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 549
 550                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 551                             NwConstants.ADD_FLOW, tx);
 552                 }
 553             }
 554         }
 555     }
 556
 557     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 558                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 559         List<MatchInfo> matches = new ArrayList<>();
 560
 561         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 562                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 563         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 564         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 565         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 566
 567         InetAddress destPrefix;
 568         try {
 569             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 570         } catch (UnknownHostException e) {
 571             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 572                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 573             return;
 574         }
 575
 576         // Match on VpnId and SubnetBroadCast IP address
 577         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 578         matches.add(MatchEthernetType.IPV4);
 579
 580         if (prefixLength != 0) {
 581             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 582         }
 583
 584         //Action is to drop the packet
 585         List<InstructionInfo> dropInstructions = new ArrayList<>();
 586         List<ActionInfo> actionsInfos = new ArrayList<>();
 587         actionsInfos.add(new ActionDrop());
 588         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 589
 590         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 591         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 592         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 593                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 594
 595         Flow flow = flowEntity.getFlowBuilder().build();
 596         String flowId = flowEntity.getFlowId();
 597         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 598         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 599
 600         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 601                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 602                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 603
 604         if (addOrRemove == NwConstants.ADD_FLOW) {
 605             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 606         } else {
 607             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 608         }
 609     }
 610
 611     /*
 612      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 613      * LportDispatcher table (via table 80)
 614      */
 615     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 616                                             final VrfEntry vrfEntry, int etherType) {
 617         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 618         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 619         // packet is commuted from Vpn2 to Vpn1.
 620         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 621         if (!interVpnLink.isActive()) {
 622             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 623                      interVpnLinkName, vrfEntry.getDestPrefix());
 624             return;
 625         }
 626
 627         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 628         if (!optLportTag.isPresent()) {
 629             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 630             return;
 631         }
 632
 633         Long lportTag = optLportTag.get();
 634         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 635         if (label == null) {
 636             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 637                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 638             return;
 639         }
 640         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 641         List<InstructionInfo> instructions = Arrays.asList(
 642             new InstructionApplyActions(actionsInfos),
 643             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 644                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 645                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 646                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 647             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 648         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 649         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 650
 651         for (BigInteger dpId : targetDpns) {
 652             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 653                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 654
 655             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 656                                /*writeTx*/null);
 657         }
 658     }
 659
 660
 661     /*
 662      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 663      */
 664     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 665                                                final VrfEntry vrfEntry, long vpnTag) {
 666         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 667         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 668             && vrfEntry.getRoutePaths().size() == 1);
 669         String destination = vrfEntry.getDestPrefix();
 670         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 671         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 672
 673         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 674         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 675         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 676         if (interVpnLink.getState().or(State.Error) != State.Active) {
 677             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 678                 destination, nextHop, interVpnLinkName);
 679             return;
 680         }
 681
 682         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 683         if (!optOtherEndpointLportTag.isPresent()) {
 684             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 685                 vpnUuid, interVpnLinkName);
 686             return;
 687         }
 688
 689         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 690         if (targetDpns.isEmpty()) {
 691             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 692                 vpnUuid, interVpnLinkName);
 693             return;
 694         }
 695
 696         String[] values = destination.split("/");
 697         String destPrefixIpAddress = values[0];
 698         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 699
 700         List<MatchInfo> matches = new ArrayList<>();
 701         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 702         matches.add(MatchEthernetType.IPV4);
 703
 704         if (prefixLength != 0) {
 705             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 706         }
 707
 708         List<Instruction> instructions =
 709             Arrays.asList(new InstructionWriteMetadata(
 710                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 711                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 712                             .L3VPN_SERVICE_INDEX)),
 713                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 714                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 715
 716         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 717         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 718         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 719             COOKIE_VM_FIB_TABLE, matches, instructions);
 720
 721         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 722             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 723
 724         for (BigInteger dpId : targetDpns) {
 725
 726             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 727                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 728                 dpId, interVpnLink.getInterVpnLinkName());
 729
 730             mdsalManager.installFlow(dpId, flowEntity);
 731         }
 732     }
 733
 734     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 735         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 736         String localNextHopIP = vrfEntry.getDestPrefix();
 737         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 738         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 739         if (localNextHopInfo == null) {
 740             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 741             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 742                     vpnName, usedRds, localNextHopIP);
 743             if (LOG.isDebugEnabled()) {
 744                 LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 745                         vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 746             }
 747             boolean localNextHopSeen = false;
 748             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 749             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 750                 String ipPrefix;
 751                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 752                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 753                 } else {
 754                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 755                 }
 756                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 757                     ipPrefix);
 758                 if (localNextHopInfoLocal != null) {
 759                     localNextHopSeen = true;
 760                     BigInteger dpnId =
 761                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 762                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 763                     returnLocalDpnId.add(dpnId);
 764                 }
 765             }
 766             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 767                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 768                 if (optionalLabel.isPresent()) {
 769                     Long label = optionalLabel.get();
 770                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 771                     synchronized (label.toString().intern()) {
 772                         LabelRouteInfo lri = getLabelRouteInfo(label);
 773                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 774                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 775                                     fibUtil.getVpnInstanceOpData(rd);
 776                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 777                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 778                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 779                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 780                                     localNextHopIP = lri.getPrefix();
 781                                 } else {
 782                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 783                                     localNextHopIP = lri.getPrefix();
 784                                 }
 785                             }
 786                             if (localNextHopInfo != null) {
 787                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 788                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 789                                 if (vpnExtraRoutes.isEmpty()) {
 790                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 791                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 792                                     returnLocalDpnId.add(dpnId);
 793                                 } else {
 794                                     for (Routes extraRoutes : vpnExtraRoutes) {
 795                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 796                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 797                                         returnLocalDpnId.add(dpnId);
 798                                     }
 799                                 }
 800                             }
 801                         }
 802                     }
 803                 }
 804             }
 805             if (returnLocalDpnId.isEmpty()) {
 806                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 807             }
 808         } else {
 809             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 810                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 811             if (dpnId != null) {
 812                 returnLocalDpnId.add(dpnId);
 813             }
 814         }
 815         return returnLocalDpnId;
 816     }
 817
 818     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 819                                                 final Long vpnId, final String rd,
 820                                                 final VrfEntry vrfEntry,
 821                                                 Routes routes, List<Routes> vpnExtraRoutes,
 822                                                 int etherType) {
 823         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 824         if (localNextHopInfo != null) {
 825             long groupId;
 826             long localGroupId;
 827             final BigInteger dpnId = localNextHopInfo.getDpnId();
 828             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 829                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 830                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 831                 return dpnId;
 832             }
 833             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 834                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 835                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 836                 return dpnId;
 837             }
 838             if (!isVpnPresentInDpn(rd, dpnId)) {
 839                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 840                         vpnId, rd, dpnId.toString());
 841                 return BigInteger.ZERO;
 842             }
 843             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 844             String prefix = vrfEntry.getDestPrefix();
 845             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 846             //The loadbalancing group is created only if the extra route has multiple nexthops
 847             //to avoid loadbalancing the discovered routes
 848             if (vpnExtraRoutes != null && routes != null) {
 849                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 850                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 851                 } else {
 852                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 853                 }
 854                 if (vpnExtraRoutes.size() > 1) {
 855                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 856                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 857                 } else if (routes.getNexthopIpList().size() > 1) {
 858                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 859                     localGroupId = groupId;
 860                 } else {
 861                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 862                             prefix, gwMacAddress);
 863                     localGroupId = groupId;
 864                 }
 865             } else {
 866                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 867                         gwMacAddress);
 868                 localGroupId = groupId;
 869             }
 870             if (groupId == FibConstants.INVALID_GROUP_ID) {
 871                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 872                         prefix, rd, interfaceName, dpnId.toString());
 873                 return BigInteger.ZERO;
 874             }
 875             final List<InstructionInfo> instructions = Collections.singletonList(
 876                     new InstructionApplyActions(
 877                             Collections.singletonList(new ActionGroup(groupId))));
 878             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 879                     new InstructionApplyActions(
 880                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(groupId))));
 881             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 882             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 883             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 884             jobCoordinator.enqueueJob(jobKey, () -> {
 885                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 886                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 887                             NwConstants.ADD_FLOW, tx, null);
 888                     if (FibUtil.isBgpVpn(vpnName, rd)) {
 889                         optLabel.ifPresent(label -> {
 890                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 891                                 LOG.debug(
 892                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 893                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 894                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 895                                         nextHopAddressList);
 896                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 897                                         NwConstants.ADD_FLOW, tx);
 898                                 // If the extra-route is reachable from VMs attached to the same switch,
 899                                 // then the tunnel table can point to the load balancing group.
 900                                 // If it is reachable from VMs attached to different switches,
 901                                 // then it should be pointing to one of the local group in order to avoid looping.
 902                                 if (vrfEntry.getRoutePaths().size() == 1) {
 903                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 904                                 } else {
 905                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 906                                 }
 907                             } else {
 908                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 909                                                 + "route. LFib and Terminating table entries will not be created.",
 910                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 911                             }
 912                         });
 913                     }
 914                 }));
 915             });
 916             return dpnId;
 917         }
 918         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 919                 vpnName);
 920         return BigInteger.ZERO;
 921     }
 922
 923     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 924         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 925         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 926         if (dpnInVpn.isPresent()) {
 927             return true;
 928         }
 929         return false;
 930     }
 931
 932     private LabelRouteInfo getLabelRouteInfo(Long label) {
 933         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 934             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 935         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 936         if (opResult.isPresent()) {
 937             return opResult.get();
 938         }
 939         return null;
 940     }
 941
 942     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 943         if (lri == null) {
 944             return true;
 945         }
 946
 947         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 948         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 949             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 950
 951         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 952             ? lri.getVpnInstanceList() : new ArrayList<>();
 953         if (vpnInstancesList.contains(vpnInstanceName)) {
 954             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 955             vpnInstancesList.remove(vpnInstanceName);
 956         }
 957         if (vpnInstancesList.isEmpty()) {
 958             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 959             if (tx != null) {
 960                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 961             } else {
 962                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 963             }
 964             return true;
 965         } else {
 966             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 967             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 968             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 969         }
 970         return false;
 971     }
 972
 973     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 974                                       WriteTransaction tx) {
 975         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 976
 977         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 978
 979         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 980             dpId, label, groupId);
 981     }
 982
 983     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 984                                                 WriteTransaction tx) {
 985         List<MatchInfo> mkMatches = new ArrayList<>();
 986
 987         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 988             destDpId, label, actionsInfos);
 989
 990         // Matching metadata
 991         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 992         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 993
 994         List<InstructionInfo> mkInstructions = new ArrayList<>();
 995         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 996
 997         FlowEntity terminatingServiceTableFlowEntity =
 998             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 999             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1000                 String.format("%s:%d", "TST Flow Entry ", label),
1001             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1002
1003         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1004
1005         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1006
1007         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1008         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1009             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1010             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1011             .child(Flow.class, flowKey).build();
1012         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
1013                 WriteTransaction.CREATE_MISSING_PARENTS);
1014     }
1015
1016     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1017         FlowEntity flowEntity;
1018         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1019         List<MatchInfo> mkMatches = new ArrayList<>();
1020         // Matching metadata
1021         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1022         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1023             NwConstants.INTERNAL_TUNNEL_TABLE,
1024             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1025             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1026             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1027         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1028         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1029         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1030             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1031             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1032
1033         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1034         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1035     }
1036
1037     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1038         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1039         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1040         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1041         boolean shouldUpdateNonEcmpLocalNextHop = true;
1042         if (localNextHopInfo == null) {
1043             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1044             if (usedRds.size() > 1) {
1045                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1046                         vrfEntry.getDestPrefix(), vpnName, rd);
1047                 return returnLocalDpnId;
1048             }
1049             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1050             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1051             //in the vpn
1052             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1053                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1054             if (extraRouteOptional.isPresent()) {
1055                 Routes extraRoute = extraRouteOptional.get();
1056                 String ipPrefix;
1057                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1058                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1059                 } else {
1060                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1061                 }
1062                 if (extraRoute.getNexthopIpList().size() > 1) {
1063                     shouldUpdateNonEcmpLocalNextHop = false;
1064                 }
1065                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1066                 if (localNextHopInfo != null) {
1067                     String localNextHopIP = localNextHopInfo.getIpAddress();
1068                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd,
1069                             vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1070                     if (!dpnId.equals(BigInteger.ZERO)) {
1071                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1072                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1073                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1074                                 /*remove*/ false);
1075                         returnLocalDpnId.add(dpnId);
1076                     }
1077                 } else {
1078                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1079                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1080                 }
1081             }
1082
1083             if (localNextHopInfo == null) {
1084                 /* Imported VRF entry */
1085                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1086                 if (optionalLabel.isPresent()) {
1087                     Long label = optionalLabel.get();
1088                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1089                     LabelRouteInfo lri = getLabelRouteInfo(label);
1090                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1091                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1092                         prefixBuilder.setDpnId(lri.getDpnId());
1093                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1094                                 vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1095                         if (!dpnId.equals(BigInteger.ZERO)) {
1096                             returnLocalDpnId.add(dpnId);
1097                         }
1098                     }
1099                 }
1100             }
1101
1102         } else {
1103             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1104             String localNextHopIP = localNextHopInfo.getIpAddress();
1105             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, vpnName, vpnId, rd, vrfEntry,
1106                     shouldUpdateNonEcmpLocalNextHop);
1107             if (!dpnId.equals(BigInteger.ZERO)) {
1108                 returnLocalDpnId.add(dpnId);
1109             }
1110         }
1111
1112         return returnLocalDpnId;
1113     }
1114
1115     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1116             final String vpnName, final Long vpnId, final String rd, final VrfEntry vrfEntry,
1117             boolean shouldUpdateNonEcmpLocalNextHop) {
1118         if (localNextHopInfo != null) {
1119             final BigInteger dpnId = localNextHopInfo.getDpnId();
1120             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1121                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1122                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1123                 return dpnId;
1124             }
1125             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1126                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1127                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1128                 return dpnId;
1129             }
1130
1131             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1132                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1133                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1134                             NwConstants.DEL_FLOW, tx, null);
1135                     if (FibUtil.isBgpVpn(vpnName, rd)) {
1136                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1137                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1138                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1139                                         NwConstants.DEL_FLOW, tx);
1140                                 removeTunnelTableEntry(dpnId, label, tx);
1141                             });
1142                         }
1143                     }
1144                 })));
1145             //TODO: verify below adjacency call need to be optimized (?)
1146             //In case of the removal of the extra route, the loadbalancing group is updated
1147             if (shouldUpdateNonEcmpLocalNextHop) {
1148                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1149             }
1150             return dpnId;
1151         }
1152         return BigInteger.ZERO;
1153     }
1154
1155     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1156             final VrfEntry vrfEntry, WriteTransaction tx) {
1157         if (tx == null) {
1158             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1159                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1160             }), LOG, "Error creating remote FIB entry");
1161             return;
1162         }
1163
1164         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1165         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1166                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1167
1168         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1169         if (adjacencyResults.isEmpty()) {
1170             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1171                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1172             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1173             return;
1174         }
1175
1176         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1177         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1178                 vpnName, usedRds, vrfEntry.getDestPrefix());
1179         // create loadbalancing groups for extra routes only when the extra route is present behind
1180         // multiple VMs
1181         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1182                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1183             List<InstructionInfo> instructions = new ArrayList<>();
1184             // Obtain the local routes for this particular dpn.
1185             java.util.Optional<Routes> routes = vpnExtraRoutes
1186                     .stream()
1187                     .filter(route -> {
1188                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1189                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1190                         if (prefixToInterface == null) {
1191                             return false;
1192                         }
1193                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1194                     }).findFirst();
1195             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1196                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1197             if (groupId == FibConstants.INVALID_GROUP_ID) {
1198                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1199                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1200                 return;
1201             }
1202             List<ActionInfo> actionInfos =
1203                     Collections.singletonList(new ActionGroup(groupId));
1204             instructions.add(new InstructionApplyActions(actionInfos));
1205             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1206                     NwConstants.ADD_FLOW, tx, null);
1207         } else {
1208             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1209         }
1210
1211         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1212     }
1213
1214     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1215     /* Get interface info from prefix to interface mapping;
1216         Use the interface info to get the corresponding vpn interface op DS entry,
1217         remove the adjacency corresponding to this fib entry.
1218         If adjacency removed is the last adjacency, clean up the following:
1219          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1220          - prefix to interface entry
1221          - vpn interface op DS
1222      */
1223         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1224         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1225         if (prefixInfo == null) {
1226             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1227             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1228             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1229             if (extraRoute != null) {
1230                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1231                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1232                     if (nextHopIp != null) {
1233                         String ipPrefix;
1234                         if (isIpv4Address(nextHopIp)) {
1235                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1236                         } else {
1237                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1238                         }
1239                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1240                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1241                     }
1242                 }
1243             }
1244             if (prefixInfo == null) {
1245                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1246                 if (optionalLabel.isPresent()) {
1247                     Long label = optionalLabel.get();
1248                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1249                     LabelRouteInfo lri = getLabelRouteInfo(label);
1250                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1251                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1252                         prefixBuilder.setDpnId(lri.getDpnId());
1253                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1254                         prefixBuilder.setIpAddress(lri.getPrefix());
1255                         prefixInfo = prefixBuilder.build();
1256                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1257                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1258                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1259                     }
1260                 }
1261             }
1262         } else {
1263             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1264         }
1265     }
1266
1267     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1268                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1269
1270         if (prefixInfo == null) {
1271             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1272                     vrfEntry.getDestPrefix(), vpnId, rd);
1273             return; //Don't have any info for this prefix (shouldn't happen); need to return
1274         }
1275
1276         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1277             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1278                     vrfEntry.getDestPrefix(), vpnId, rd);
1279             return;
1280         }
1281
1282         String ifName = prefixInfo.getVpnInterfaceName();
1283         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1284             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1285     }
1286
1287     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1288         Prefixes prefixInfo;
1289         Long vpnId;
1290         String rd;
1291         VrfEntry vrfEntry;
1292         Routes extraRoute;
1293
1294         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1295                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1296             this.prefixInfo = prefixInfo;
1297             this.vpnId = vpnId;
1298             this.rd = rd;
1299             this.vrfEntry = vrfEntry;
1300             this.extraRoute = extraRoute;
1301         }
1302
1303         @Override
1304         public List<ListenableFuture<Void>> call() {
1305             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1306             // to call the respective helpers.
1307             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1308                 //First Cleanup LabelRouteInfo
1309                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1310                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1311                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1312                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1313                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1314                         synchronized (label.toString().intern()) {
1315                             LabelRouteInfo lri = getLabelRouteInfo(label);
1316                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1317                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1318                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1319                                         fibUtil.getVpnInstanceOpData(rd);
1320                                 String vpnInstanceName = "";
1321                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1322                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1323                                 }
1324                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1325                                 if (lriRemoved) {
1326                                     String parentRd = lri.getParentVpnRd();
1327                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1328                                             parentRd, vrfEntry.getDestPrefix()));
1329                                 }
1330                             } else {
1331                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1332                                         rd, vrfEntry.getDestPrefix()));
1333                             }
1334                         }
1335                     });
1336                 }
1337                 String ifName = prefixInfo.getVpnInterfaceName();
1338                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1339                 String vpnName = null;
1340
1341                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1342                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1343                     * */
1344                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1345                     if (vpnNameOpt.isPresent()) {
1346                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1347                     }
1348                 }
1349                 if (optVpnName.isPresent()) {
1350                     vpnName = optVpnName.get();
1351                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1352                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1353                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1354                     if (opVpnInterface.isPresent()) {
1355                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1356                         if (vpnId != associatedVpnId) {
1357                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1358                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1359                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1360                             return;
1361                         } else {
1362                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1363                                     vrfEntry.getDestPrefix(), associatedVpnId);
1364                         }
1365                     }
1366                 }
1367                 if (extraRoute != null) {
1368                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1369                     //Only one used Rd present in case of removal event
1370                     String usedRd = usedRds.get(0);
1371                     if (optVpnName.isPresent()) {
1372                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1373                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1374                                         vrfEntry.getDestPrefix()));
1375                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1376                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1377                     }
1378                 }
1379                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1380             }));
1381         }
1382     }
1383
1384     /**
1385      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1386      * Remove Adjacency from VPNInterfaceOpData.
1387      * if Adjacency != primary.
1388      * if Adjacency == primary , then mark it for deletion.
1389      * Remove entire VPNinterfaceOpData Entry.
1390      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1391      * @param vrfEntry - VrfEntry removed
1392      * @param ifName - Interface name from VRFentry
1393      * @param vpnName - VPN name of corresponding VRF
1394      * @param tx - ReadWrite Tx
1395      * @throws ReadFailedException - Exception thrown in case of read failed
1396      */
1397     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1398                                                           ReadWriteTransaction tx) throws ReadFailedException {
1399         InstanceIdentifier<Adjacency> adjacencyIid =
1400                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1401         Optional<Adjacency> adjacencyOptional = tx.read(LogicalDatastoreType.OPERATIONAL, adjacencyIid).checkedGet();
1402         if (adjacencyOptional.isPresent()) {
1403             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1404                 tx.delete(LogicalDatastoreType.OPERATIONAL,
1405                         FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1406             } else {
1407                 tx.merge(LogicalDatastoreType.OPERATIONAL, adjacencyIid,
1408                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1409             }
1410         }
1411
1412         Optional<AdjacenciesOp> optAdjacencies =
1413                 tx.read(LogicalDatastoreType.OPERATIONAL,
1414                         FibUtil.getAdjListPathOp(ifName, vpnName)).checkedGet();
1415
1416         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1417             return;
1418         }
1419
1420         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1421                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1422                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1423                         && adjacency.isMarkedForDeletion() != null
1424                         && adjacency.isMarkedForDeletion()
1425         )) {
1426             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1427             tx.delete(LogicalDatastoreType.OPERATIONAL,
1428                     FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1429         }
1430     }
1431
1432     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1433         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1434         final String rd = vrfTableKey.getRouteDistinguisher();
1435         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1436         if (vpnInstance == null) {
1437             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1438             return;
1439         }
1440         final Collection<VpnToDpnList> vpnToDpnList;
1441         if (vrfEntry.getParentVpnRd() != null
1442                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1443             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1444             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1445             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1446                     vpnInstance.getVpnToDpnList();
1447             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1448                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1449         } else {
1450             vpnToDpnList = vpnInstance.getVpnToDpnList();
1451         }
1452
1453         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1454         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1455         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1456         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1457         if (subnetRoute != null) {
1458             long elanTag = subnetRoute.getElantag();
1459             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1460                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1461             if (vpnToDpnList != null) {
1462                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1463                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1464                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1465
1466                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1467                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1468                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1469                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1470                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1471                             }
1472
1473                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1474                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1475                         }
1476                     })));
1477             }
1478             optionalLabel.ifPresent(label -> {
1479                 synchronized (label.toString().intern()) {
1480                     LabelRouteInfo lri = getLabelRouteInfo(label);
1481                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1482                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1483                                 fibUtil.getVpnInstanceOpData(rd);
1484                         String vpnInstanceName = "";
1485                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1486                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1487                         }
1488                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1489                         if (lriRemoved) {
1490                             String parentRd = lri.getParentVpnRd();
1491                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1492                                     parentRd, vrfEntry.getDestPrefix()));
1493                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1494                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1495                         }
1496                     } else {
1497                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1498                                 rd, vrfEntry.getDestPrefix()));
1499                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1500                                 label, rd, vrfEntry.getDestPrefix());
1501                     }
1502                 }
1503             });
1504             return;
1505         }
1506
1507         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1508             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1509         if (vpnToDpnList != null) {
1510             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1511                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1512             String jobKey;
1513             Optional<Routes> extraRouteOptional;
1514             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1515             if (usedRds != null && !usedRds.isEmpty()) {
1516                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1517                 if (usedRds.size() > 1) {
1518                     LOG.error("The extra route prefix is still present in some DPNs");
1519                     return ;
1520                 } else {
1521                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1522                     //is not present in any other DPN
1523                     extraRouteOptional = VpnExtraRouteHelper
1524                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1525                 }
1526             } else {
1527                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1528                 extraRouteOptional = Optional.absent();
1529             }
1530
1531             jobCoordinator.enqueueJob(jobKey,
1532                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1533                     if (localDpnIdList.size() <= 0) {
1534                         for (VpnToDpnList curDpn : vpnToDpnList) {
1535                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1536                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1537                         }
1538                     } else {
1539                         for (BigInteger localDpnId : localDpnIdList) {
1540                             for (VpnToDpnList curDpn : vpnToDpnList) {
1541                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1542                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1543                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1544                                 }
1545                             }
1546                         }
1547                     }
1548                 })), MAX_RETRIES);
1549         }
1550
1551         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1552         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1553         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1554
1555         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1556         // of the interVpnLink.
1557         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1558         if (optVpnUuid.isPresent()) {
1559             String vpnUuid = optVpnUuid.get();
1560             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1561                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1562                 if (optInterVpnLink.isPresent()) {
1563                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1564                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1565                         // This is route that points to the other endpoint of an InterVpnLink
1566                         // In that case, we should look for the FIB table pointing to
1567                         // LPortDispatcher table and remove it.
1568                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1569                     }
1570                 }
1571             });
1572         }
1573
1574     }
1575
1576     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1577                                     int addOrRemove, WriteTransaction tx) {
1578         if (tx == null) {
1579             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1580                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1581                 "Error making LFIB table entry");
1582             return;
1583         }
1584
1585         List<MatchInfo> matches = new ArrayList<>();
1586         matches.add(MatchEthernetType.MPLS_UNICAST);
1587         matches.add(new MatchMplsLabel(label));
1588
1589         // Install the flow entry in L3_LFIB_TABLE
1590         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1591
1592         FlowEntity flowEntity;
1593         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1594             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1595         Flow flow = flowEntity.getFlowBuilder().build();
1596         String flowId = flowEntity.getFlowId();
1597         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1598         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1599         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1600             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1601             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1602
1603         if (addOrRemove == NwConstants.ADD_FLOW) {
1604             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1605         } else {
1606             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1607         }
1608
1609         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1610             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1611     }
1612
1613     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1614                                     final FutureCallback<List<Void>> callback) {
1615         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1616         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1617             () -> {
1618                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1619                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1620                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1621                         LogicalDatastoreType.CONFIGURATION, id);
1622                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1623                 if (!vrfTable.isPresent()) {
1624                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1625                     if (callback != null) {
1626                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1627                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1628                     }
1629                     return futures;
1630                 }
1631                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1632                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1633                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1634                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1635                             if (subnetRoute != null) {
1636                                 long elanTag = subnetRoute.getElantag();
1637                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1638                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1639                                         tx);
1640                                 continue;
1641                             }
1642                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1643                             if (routerInt != null) {
1644                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1645                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1646                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1647                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1648                                         NwConstants.ADD_FLOW);
1649                                 continue;
1650                             }
1651                             //Handle local flow creation for imports
1652                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1653                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1654                                 if (optionalLabel.isPresent()) {
1655                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1656                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1657                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1658                                         if (lri.getDpnId().equals(dpnId)) {
1659                                             try {
1660                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1661                                                         vrfEntry.getDestPrefix());
1662                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1663                                             } catch (IllegalArgumentException ex) {
1664                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1665                                                         vrfEntry.getDestPrefix());
1666                                             }
1667                                             continue;
1668                                         }
1669                                     }
1670                                 }
1671                             }
1672                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1673                                     vrfEntry, dpnId);
1674                             if (shouldCreateRemoteFibEntry) {
1675                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1676                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1677                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1678                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1679                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1680                                 } else {
1681                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1682                                             vrfEntry, tx);
1683                                 }
1684                             }
1685                         }
1686                     }));
1687                     if (callback != null) {
1688                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1689                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1690                     }
1691                 }
1692                 return futures;
1693             });
1694     }
1695
1696     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1697                                             final String localNextHopIp, final String remoteNextHopIp) {
1698         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1699             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1700         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1701         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1702         List<SubTransaction> txnObjects =  new ArrayList<>();
1703         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1704         if (vrfTable.isPresent()) {
1705             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1706                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1707                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1708                         vrfTable.get().getVrfEntry().stream()
1709                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1710                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1711                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1712                     }
1713                 })));
1714         }
1715     }
1716
1717     public void manageRemoteRouteOnDPN(final boolean action,
1718                                        final BigInteger localDpnId,
1719                                        final long vpnId,
1720                                        final String rd,
1721                                        final String destPrefix,
1722                                        final String destTepIp,
1723                                        final long label) {
1724         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1725
1726         if (vpnInstance == null) {
1727             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1728             return;
1729         }
1730
1731         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1732             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1733                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1734                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1735                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1736                     if (vrfEntry == null) {
1737                         return;
1738                     }
1739                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1740                             action, localDpnId, vpnId, rd, destPrefix);
1741                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1742                     VrfEntry modVrfEntry;
1743                     if (routePathList == null || routePathList.isEmpty()) {
1744                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1745                                 Collections.singletonList(destTepIp),
1746                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1747                     } else {
1748                         modVrfEntry = vrfEntry;
1749                     }
1750
1751                     if (action) {
1752                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1753                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1754                                 modVrfEntry, tx);
1755                     } else {
1756                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1757                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1758                                 vrfEntry.getDestPrefix());
1759                         if (usedRds.size() > 1) {
1760                             LOG.debug("The extra route prefix is still present in some DPNs");
1761                             return;
1762                         }
1763                         //Is this fib route an extra route? If yes, get the nexthop which would be
1764                         //an adjacency in the vpn
1765                         Optional<Routes> extraRouteOptional = Optional.absent();
1766                         if (usedRds.size() != 0) {
1767                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1768                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1769                                     usedRds.get(0), vrfEntry.getDestPrefix());
1770                         }
1771                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1772                                 extraRouteOptional, tx);
1773                     }
1774                 }
1775             })));
1776     }
1777
1778     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1779                                  final FutureCallback<List<Void>> callback) {
1780         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1781         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1782             () -> {
1783                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1784                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1785                 List<SubTransaction> txnObjects =  new ArrayList<>();
1786                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1787                         LogicalDatastoreType.CONFIGURATION, id);
1788                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1789                 if (vrfTable.isPresent()) {
1790                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1791                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1792                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1793                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1794                                 /* Handle subnet routes here */
1795                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1796                                 if (subnetRoute != null && !fibUtil
1797                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1798                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1799                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1800                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1801                                             NwConstants.DEL_FLOW, tx, null);
1802                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1803                                     if (routePaths != null) {
1804                                         for (RoutePaths routePath : routePaths) {
1805                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1806                                                     DEFAULT_FIB_FLOW_PRIORITY,
1807                                                     NwConstants.DEL_FLOW, tx);
1808                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1809                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1810                                                     vrfEntry.getDestPrefix());
1811                                         }
1812                                     }
1813                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1814                                             NwConstants.DEL_FLOW, tx);
1815                                     continue;
1816                                 }
1817                                 // ping responder for router interfaces
1818                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1819                                 if (routerInt != null) {
1820                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1821                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1822                                             routerInt.getMacAddress());
1823                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1824                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1825                                             NwConstants.DEL_FLOW);
1826                                     continue;
1827                                 }
1828
1829                                 //Handle local flow deletion for imports
1830                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1831                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1832                                     if (optionalLabel.isPresent()) {
1833                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1834                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1835                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1836                                                 lri) && lri.getDpnId().equals(dpnId)) {
1837                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1838                                         }
1839                                     }
1840                                 }
1841
1842                                 // Passing null as we don't know the dpn
1843                                 // to which prefix is attached at this point
1844                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1845                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1846                                 Optional<Routes> extraRouteOptional;
1847                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1848                                 //an adjacency in the vpn
1849                                 if (usedRds != null && !usedRds.isEmpty()) {
1850                                     if (usedRds.size() > 1) {
1851                                         LOG.error("The extra route prefix is still present in some DPNs");
1852                                         return;
1853                                     } else {
1854                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1855                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1856
1857                                     }
1858                                 } else {
1859                                     extraRouteOptional = Optional.absent();
1860                                 }
1861                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1862                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1863                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1864                                 } else {
1865                                     if (subnetRoute == null || !fibUtil
1866                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1867                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1868                                                 vrfTable.get().key(), vrfEntry, extraRouteOptional, tx);
1869                                     }
1870                                 }
1871                             }
1872                         }));
1873                     }
1874                     if (callback != null) {
1875                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1876                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1877                     }
1878                 } else {
1879                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1880                 }
1881                 return futures;
1882             });
1883
1884     }
1885
1886     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1887                                            final String localNextHopIp, final String remoteNextHopIp) {
1888         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1889                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1890             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1891         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1892         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1893         List<SubTransaction> txnObjects =  new ArrayList<>();
1894         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1895         if (vrfTable.isPresent()) {
1896             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1897                 () -> {
1898                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1899                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1900                             tx -> vrfTable.get().getVrfEntry().stream()
1901                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1902                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1903                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1904                     }
1905                 });
1906         }
1907     }
1908
1909     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1910         InstanceIdentifierBuilder<VrfTables> idBuilder =
1911             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1912         return idBuilder.build();
1913     }
1914
1915     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1916         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1917                 .FLOWID_SEPARATOR + nextHop;
1918     }
1919
1920     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1921         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1922                 + tableMiss + FLOWID_PREFIX;
1923     }
1924
1925     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1926         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1927             .child(VrfTables.class, new VrfTablesKey(rd))
1928             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1929         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1930         if (vrfEntry.isPresent()) {
1931             return vrfEntry.get();
1932         }
1933         return null;
1934     }
1935
1936     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1937                                              final String vpnName,
1938                                              final VrfEntry vrfEntry) {
1939         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1940
1941         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1942         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1943
1944         if (targetDpns.isEmpty()) {
1945             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1946             return;
1947         }
1948
1949         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1950         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1951
1952         // delete from FIB
1953         //
1954         optNextHop.ifPresent(nextHop -> {
1955             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1956             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1957             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1958                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1959
1960             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1961             for (BigInteger dpId : targetDpns) {
1962                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1963                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1964
1965                 mdsalManager.removeFlow(dpId, flow);
1966             }
1967         });
1968
1969         // delete from LFIB
1970         //
1971         optLabel.ifPresent(label -> {
1972             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1973
1974             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1975                 for (BigInteger dpId : targetDpns) {
1976                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1977                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1978                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1979                             tx);
1980                 }
1981             }), LOG, "Error removing flows");
1982         });
1983     }
1984
1985     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1986             List<String> nextHopAddressList, LabelRouteInfo lri) {
1987         return lri != null && lri.getPrefix().equals(prefix)
1988                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1989     }
1990
1991     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1992         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1993             return true;
1994         }
1995
1996         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1997         if (prefix != null) {
1998             BigInteger prefixDpnId = prefix.getDpnId();
1999             if (dpnId.equals(prefixDpnId)) {
2000                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2001                         vrfEntry, dpnId);
2002                 return false;
2003             }
2004         }
2005         return true;
2006     }
2007 }