fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
  42 import org.opendaylight.genius.mdsalutil.ActionInfo;
  43 import org.opendaylight.genius.mdsalutil.FlowEntity;
  44 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  48 import org.opendaylight.genius.mdsalutil.NWUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.UpgradeState;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  54 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  57 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  63 import org.opendaylight.genius.utils.ServiceIndex;
  64 import org.opendaylight.genius.utils.batching.SubTransaction;
  65 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  66 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  67 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  68 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  69 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  70 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  71 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  72 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  73 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  74 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 109 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 110 import org.slf4j.Logger;
 111 import org.slf4j.LoggerFactory;
 112
 113
 114 @Singleton
 115 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 116
 117     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 118     private static final String FLOWID_PREFIX = "L3.";
 119     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 120     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 121     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 122     private static final int LFIB_INTERVPN_PRIORITY = 15;
 123     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 124     private static final int MAX_RETRIES = 3;
 125     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 126
 127     private final DataBroker dataBroker;
 128     private final ManagedNewTransactionRunner txRunner;
 129     private final RetryingManagedNewTransactionRunner retryingTxRunner;
 130     private final IMdsalApiManager mdsalManager;
 131     private final NexthopManager nextHopManager;
 132     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 133     private final BaseVrfEntryHandler baseVrfEntryHandler;
 134     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 135     private final JobCoordinator jobCoordinator;
 136     private final IElanService elanManager;
 137     private final FibUtil fibUtil;
 138     private final InterVpnLinkCache interVpnLinkCache;
 139     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 140     private final UpgradeState upgradeState;
 141     private final DataTreeEventCallbackRegistrar eventCallbacks;
 142
 143     @Inject
 144     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 145                             final NexthopManager nexthopManager,
 146                             final IElanService elanManager,
 147                             final BaseVrfEntryHandler vrfEntryHandler,
 148                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 149                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 150                             final JobCoordinator jobCoordinator,
 151                             final FibUtil fibUtil,
 152                             final InterVpnLinkCache interVpnLinkCache,
 153                             final UpgradeState upgradeState,
 154                             final DataTreeEventCallbackRegistrar eventCallbacks) {
 155         super(VrfEntry.class, VrfEntryListener.class);
 156         this.dataBroker = dataBroker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 158         this.retryingTxRunner = new RetryingManagedNewTransactionRunner(dataBroker, MAX_RETRIES);
 159         this.mdsalManager = mdsalApiManager;
 160         this.nextHopManager = nexthopManager;
 161         this.elanManager = elanManager;
 162         this.baseVrfEntryHandler = vrfEntryHandler;
 163         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 164         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.fibUtil = fibUtil;
 167         this.interVpnLinkCache = interVpnLinkCache;
 168         this.upgradeState = upgradeState;
 169         this.eventCallbacks = eventCallbacks;
 170     }
 171
 172     @Override
 173     @PostConstruct
 174     public void init() {
 175         LOG.info("{} init", getClass().getSimpleName());
 176         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 177     }
 178
 179     @Override
 180     @SuppressWarnings("checkstyle:IllegalCatch")
 181     public void close() {
 182         closeables.forEach(c -> {
 183             try {
 184                 c.close();
 185             } catch (Exception e) {
 186                 LOG.warn("Error closing {}", c, e);
 187             }
 188         });
 189     }
 190
 191     @Override
 192     protected VrfEntryListener getDataTreeChangeListener() {
 193         return VrfEntryListener.this;
 194     }
 195
 196     @Override
 197     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 198         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 199     }
 200
 201     @Override
 202     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 203         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 204         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 205         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 206                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 207         addFibEntries(identifier, vrfEntry, rd);
 208         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 209                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 210     }
 211
 212     //This method is temporary. Eventually Factory design pattern will be used to get
 213     // right VrfEntryhandle and invoke its methods.
 214     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 215         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 216             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 217             return;
 218         }
 219         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 220             LOG.info("EVPN flows need to be programmed.");
 221             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 222                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 223             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 224             closeables.add(evpnVrfEntryHandler);
 225             return;
 226         }
 227         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 228         if (routerInt != null) {
 229             // ping responder for router interfaces
 230             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 231             return;
 232         }
 233         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 234             createFibEntries(identifier, vrfEntry);
 235             return;
 236         }
 237     }
 238
 239     @Override
 240     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 241         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 242         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 243         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 244                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 245         removeFibEntries(identifier, vrfEntry, rd);
 246         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 247             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 248     }
 249
 250     //This method is temporary. Eventually Factory design pattern will be used to get
 251     // right VrfEntryhandle and invoke its methods.
 252     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 253         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 254             LOG.info("EVPN flows to be deleted");
 255             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 256                     nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
 257             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 258             closeables.add(evpnVrfEntryHandler);
 259             return;
 260         }
 261         RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
 262         if (routerInt != null) {
 263             // ping responder for router interfaces
 264             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 265             return;
 266         }
 267         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 268             deleteFibEntries(identifier, vrfEntry);
 269             return;
 270         }
 271         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 272             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 273             return;
 274         }
 275     }
 276
 277     @Override
 278     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 279     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 280     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 281     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 282         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 283         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 284         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 285                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 286         // Handle BGP Routes first
 287         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 288             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 289             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 290                     rd, update.getDestPrefix(), update.getRoutePaths());
 291             return;
 292         }
 293
 294         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 295             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 296             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 297             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 298
 299             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 300             //updated vrf entry route path is null. This can happen during tunnel events.
 301             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 302             List<String> usedRds = new ArrayList<>();
 303             if (optVpnInstance.isPresent()) {
 304                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 305                         update.getDestPrefix());
 306             }
 307             // If original VRF Entry had nexthop null , but update VRF Entry
 308             // has nexthop , route needs to be created on remote Dpns
 309             if (originalRoutePath == null || originalRoutePath.isEmpty()
 310                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 311                 // TODO(vivek): Though ugly, Not handling this code now, as each
 312                 // tep add event will invoke flow addition
 313                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 314                         + " This event is IGNORED here.", update.getDestPrefix());
 315                 return;
 316             }
 317
 318             // If original VRF Entry had valid nexthop , but update VRF Entry
 319             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 320             if (updateRoutePath == null || updateRoutePath.isEmpty()
 321                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 322                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 323                         + "This event is IGNORED here.", update.getDestPrefix());
 324                 return;
 325             }
 326             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 327             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 328             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 329             List<ListenableFuture<Void>> futures = new ArrayList<>();
 330             ListenableFuture<Void> configFuture =
 331                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(configTx ->
 332                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
 333                                     nextHopsRemoved.parallelStream()
 334                                             .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 335                                                     configTx, operTx, nextHopRemoved, rd, update.getDestPrefix())))));
 336             futures.add(configFuture);
 337             Futures.addCallback(configFuture, new FutureCallback<Void>() {
 338                 @Override
 339                 public void onSuccess(Void result) {
 340                     createFibEntries(identifier, update);
 341                     LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 342                             rd, update.getDestPrefix(), update.getRoutePaths());
 343                 }
 344
 345                 @Override
 346                 public void onFailure(Throwable throwable) {
 347                     LOG.error("Exception encountered while submitting operational future for update vrfentry {}",
 348                             update, throwable);
 349                 }
 350             }, MoreExecutors.directExecutor());
 351             return;
 352         }
 353
 354         //Handle all other routes only on a cluster reboot
 355         if (original.equals(update)) {
 356             //Reboot use-case
 357             createFibEntries(identifier, update);
 358             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 359                     rd, update.getDestPrefix(), update.getRoutePaths());
 360             return;
 361         }
 362
 363         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 364                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 365     }
 366
 367     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 368         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 369         List<SubTransaction> txnObjects =  new ArrayList<>();
 370         final VpnInstanceOpDataEntry vpnInstance =
 371                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 372         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 373         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 374                 + " has null vpnId!");
 375         final Collection<VpnToDpnList> vpnToDpnList;
 376         if (vrfEntry.getParentVpnRd() != null
 377                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 378             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 379             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 380             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 381                 vpnInstance.getVpnToDpnList();
 382             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 383                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 384         } else {
 385             vpnToDpnList = vpnInstance.getVpnToDpnList();
 386         }
 387         final Long vpnId = vpnInstance.getVpnId();
 388         final String rd = vrfTableKey.getRouteDistinguisher();
 389         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
 390         if (subnetRoute != null) {
 391             final long elanTag = subnetRoute.getElantag();
 392             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 393                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 394             if (vpnToDpnList != null) {
 395                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 396                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 397                         for (final VpnToDpnList curDpn : vpnToDpnList) {
 398                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 399                                 installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 400                                 installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 401                                         vrfEntry, NwConstants.ADD_FLOW, tx);
 402                             }
 403                         }
 404                     })));
 405             }
 406             return;
 407         }
 408         // Get etherType value based on the IpPrefix address family type
 409         int etherType;
 410         try {
 411             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 412         } catch (IllegalArgumentException ex) {
 413             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 414             return;
 415         }
 416
 417         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType);
 418         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 419             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
 420                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 421                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
 422                         for (VpnToDpnList vpnDpn : vpnToDpnList) {
 423                             if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 424                                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 425                                     try {
 426                                         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 427                                             bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 428                                                     vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx,
 429                                                     txnObjects);
 430                                         } else {
 431                                             createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 432                                                     vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 433                                         }
 434                                     } catch (NullPointerException e) {
 435                                         LOG.error("Failed to get create remote fib flows for prefix {} ",
 436                                                 vrfEntry.getDestPrefix(), e);
 437                                     }
 438                                 }
 439                             }
 440                         }
 441                     }
 442                 })), MAX_RETRIES);
 443         }
 444
 445         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 446         if (optVpnUuid.isPresent()) {
 447             String vpnUuid = optVpnUuid.get();
 448             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 449             if (interVpnLink != null) {
 450                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 451                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 452                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 453                         // This is an static route that points to the other endpoint of an InterVpnLink
 454                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 455                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 456                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry, etherType);
 457                     }
 458                 });
 459             }
 460         }
 461     }
 462
 463     void refreshFibTables(String rd, String prefix) {
 464         InstanceIdentifier<VrfEntry> vrfEntryId =
 465                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 466                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 467         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 468         if (vrfEntry.isPresent()) {
 469             createFibEntries(vrfEntryId, vrfEntry.get());
 470         }
 471     }
 472
 473     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 474         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 475         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 476         prefixBuilder.setDpnId(lri.getDpnId());
 477         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 478         prefixBuilder.setIpAddress(lri.getPrefix());
 479         // Increment the refCount here
 480         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 481             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 482         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 483         if (!isPresentInList) {
 484             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 485             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 486             vpnInstanceNames.add(vpnInstanceName);
 487             builder.setVpnInstanceList(vpnInstanceNames);
 488             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 489         } else {
 490             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 491         }
 492         return prefixBuilder.build();
 493     }
 494
 495     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 496                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 497         if (tx == null) {
 498             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 499                 newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG,
 500                 "Error installing subnet route in FIB");
 501             return;
 502         }
 503         int etherType;
 504         try {
 505             etherType = NWUtil.getEtherTypeFromIpPrefix(vrfEntry.getDestPrefix());
 506         } catch (IllegalArgumentException ex) {
 507             LOG.error("Unable to get etherType for IP Prefix {}", vrfEntry.getDestPrefix());
 508             return;
 509         }
 510         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 511             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 512             synchronized (label.toString().intern()) {
 513                 LabelRouteInfo lri = getLabelRouteInfo(label);
 514                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 515
 516                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 517                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 518                                 fibUtil.getVpnInstanceOpData(rd);
 519                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 520                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 521                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 522                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 523                             }
 524                         }
 525                     }
 526                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 527                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 528                 }
 529             }
 530         });
 531         final List<InstructionInfo> instructions = new ArrayList<>();
 532         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 533             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 534         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 535         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 536         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 537                 NwConstants.ADD_FLOW, tx, null);
 538         if (vrfEntry.getRoutePaths() != null) {
 539             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 540                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 541                     List<ActionInfo> actionsInfos = new ArrayList<>();
 542                     // reinitialize instructions list for LFIB Table
 543                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 544                     actionsInfos.add(new ActionPopMpls(etherType));
 545                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 546                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 547                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 548                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 549
 550                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 551                             NwConstants.ADD_FLOW, tx);
 552                 }
 553             }
 554         }
 555     }
 556
 557     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 558                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 559         List<MatchInfo> matches = new ArrayList<>();
 560
 561         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 562                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 563         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 564         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 565         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 566
 567         InetAddress destPrefix;
 568         try {
 569             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 570         } catch (UnknownHostException e) {
 571             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 572                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 573             return;
 574         }
 575
 576         // Match on VpnId and SubnetBroadCast IP address
 577         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 578         matches.add(MatchEthernetType.IPV4);
 579
 580         if (prefixLength != 0) {
 581             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 582         }
 583
 584         //Action is to drop the packet
 585         List<InstructionInfo> dropInstructions = new ArrayList<>();
 586         List<ActionInfo> actionsInfos = new ArrayList<>();
 587         actionsInfos.add(new ActionDrop());
 588         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 589
 590         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 591         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 592         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 593                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 594
 595         Flow flow = flowEntity.getFlowBuilder().build();
 596         String flowId = flowEntity.getFlowId();
 597         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 598         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 599
 600         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 601                 .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
 602                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 603
 604         if (addOrRemove == NwConstants.ADD_FLOW) {
 605             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 606         } else {
 607             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 608         }
 609     }
 610
 611     /*
 612      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 613      * LportDispatcher table (via table 80)
 614      */
 615     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 616                                             final VrfEntry vrfEntry, int etherType) {
 617         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 618         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 619         // packet is commuted from Vpn2 to Vpn1.
 620         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 621         if (!interVpnLink.isActive()) {
 622             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 623                      interVpnLinkName, vrfEntry.getDestPrefix());
 624             return;
 625         }
 626
 627         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 628         if (!optLportTag.isPresent()) {
 629             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 630             return;
 631         }
 632
 633         Long lportTag = optLportTag.get();
 634         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 635         if (label == null) {
 636             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 637                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 638             return;
 639         }
 640         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls(etherType));
 641         List<InstructionInfo> instructions = Arrays.asList(
 642             new InstructionApplyActions(actionsInfos),
 643             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 644                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 645                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 646                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 647             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 648         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 649         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 650
 651         for (BigInteger dpId : targetDpns) {
 652             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 653                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 654
 655             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 656                                /*writeTx*/null);
 657         }
 658     }
 659
 660
 661     /*
 662      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 663      */
 664     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 665                                                final VrfEntry vrfEntry, long vpnTag) {
 666         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 667         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 668             && vrfEntry.getRoutePaths().size() == 1);
 669         String destination = vrfEntry.getDestPrefix();
 670         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 671         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 672
 673         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 674         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 675         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 676         if (interVpnLink.getState().or(State.Error) != State.Active) {
 677             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 678                 destination, nextHop, interVpnLinkName);
 679             return;
 680         }
 681
 682         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 683         if (!optOtherEndpointLportTag.isPresent()) {
 684             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 685                 vpnUuid, interVpnLinkName);
 686             return;
 687         }
 688
 689         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 690         if (targetDpns.isEmpty()) {
 691             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 692                 vpnUuid, interVpnLinkName);
 693             return;
 694         }
 695
 696         String[] values = destination.split("/");
 697         String destPrefixIpAddress = values[0];
 698         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 699
 700         List<MatchInfo> matches = new ArrayList<>();
 701         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 702         matches.add(MatchEthernetType.IPV4);
 703
 704         if (prefixLength != 0) {
 705             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 706         }
 707
 708         List<Instruction> instructions =
 709             Arrays.asList(new InstructionWriteMetadata(
 710                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 711                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 712                             .L3VPN_SERVICE_INDEX)),
 713                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 714                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 715
 716         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 717         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 718         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 719             COOKIE_VM_FIB_TABLE, matches, instructions);
 720
 721         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 722             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 723
 724         for (BigInteger dpId : targetDpns) {
 725
 726             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 727                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 728                 dpId, interVpnLink.getInterVpnLinkName());
 729
 730             mdsalManager.installFlow(dpId, flowEntity);
 731         }
 732     }
 733
 734     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) {
 735         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 736         String localNextHopIP = vrfEntry.getDestPrefix();
 737         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 738         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 739         if (localNextHopInfo == null) {
 740             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 741             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 742                     vpnName, usedRds, localNextHopIP);
 743             if (LOG.isDebugEnabled()) {
 744                 LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}",
 745                         vpnName, usedRds, localNextHopIP, vpnExtraRoutes);
 746             }
 747             boolean localNextHopSeen = false;
 748             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 749             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 750                 String ipPrefix;
 751                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 752                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 753                 } else {
 754                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 755                 }
 756                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 757                     ipPrefix);
 758                 if (localNextHopInfoLocal != null) {
 759                     localNextHopSeen = true;
 760                     BigInteger dpnId =
 761                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 762                                     vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType);
 763                     returnLocalDpnId.add(dpnId);
 764                 }
 765             }
 766             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 767                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 768                 if (optionalLabel.isPresent()) {
 769                     Long label = optionalLabel.get();
 770                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 771                     synchronized (label.toString().intern()) {
 772                         LabelRouteInfo lri = getLabelRouteInfo(label);
 773                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 774                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 775                                     fibUtil.getVpnInstanceOpData(rd);
 776                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 777                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 778                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 779                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 780                                     localNextHopIP = lri.getPrefix();
 781                                 } else {
 782                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 783                                     localNextHopIP = lri.getPrefix();
 784                                 }
 785                             }
 786                             if (localNextHopInfo != null) {
 787                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 788                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 789                                 if (vpnExtraRoutes.isEmpty()) {
 790                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 791                                             vpnId, rd, vrfEntry, null, vpnExtraRoutes, etherType);
 792                                     returnLocalDpnId.add(dpnId);
 793                                 } else {
 794                                     for (Routes extraRoutes : vpnExtraRoutes) {
 795                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 796                                                 vpnId, rd, vrfEntry, extraRoutes, vpnExtraRoutes, etherType);
 797                                         returnLocalDpnId.add(dpnId);
 798                                     }
 799                                 }
 800                             }
 801                         }
 802                     }
 803                 }
 804             }
 805             if (returnLocalDpnId.isEmpty()) {
 806                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 807             }
 808         } else {
 809             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 810                     rd, vrfEntry, /*routes*/ null, /*vpnExtraRoutes*/ null, etherType);
 811             if (dpnId != null) {
 812                 returnLocalDpnId.add(dpnId);
 813             }
 814         }
 815         return returnLocalDpnId;
 816     }
 817
 818     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 819                                                 final Long vpnId, final String rd,
 820                                                 final VrfEntry vrfEntry,
 821                                                 Routes routes, List<Routes> vpnExtraRoutes,
 822                                                 int etherType) {
 823         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 824         if (localNextHopInfo != null) {
 825             long groupId;
 826             long localGroupId;
 827             final BigInteger dpnId = localNextHopInfo.getDpnId();
 828             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 829                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 830                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 831                 return dpnId;
 832             }
 833             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 834                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 835                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 836                 return dpnId;
 837             }
 838             if (!isVpnPresentInDpn(rd, dpnId)) {
 839                 LOG.error("checkCreateLocalFibEntry: The VPN with id {} rd {} is not available on dpn {}",
 840                         vpnId, rd, dpnId.toString());
 841                 return BigInteger.ZERO;
 842             }
 843             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 844             String prefix = vrfEntry.getDestPrefix();
 845             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 846             //The loadbalancing group is created only if the extra route has multiple nexthops
 847             //to avoid loadbalancing the discovered routes
 848             if (vpnExtraRoutes != null && routes != null) {
 849                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 850                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 851                 } else {
 852                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 853                 }
 854                 if (vpnExtraRoutes.size() > 1) {
 855                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 856                     localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP);
 857                 } else if (routes.getNexthopIpList().size() > 1) {
 858                     groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes);
 859                     localGroupId = groupId;
 860                 } else {
 861                     groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP,
 862                             prefix, gwMacAddress);
 863                     localGroupId = groupId;
 864                 }
 865             } else {
 866                 groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix,
 867                         gwMacAddress);
 868                 localGroupId = groupId;
 869             }
 870             if (groupId == FibConstants.INVALID_GROUP_ID) {
 871                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 872                         prefix, rd, interfaceName, dpnId.toString());
 873                 return BigInteger.ZERO;
 874             }
 875             final List<InstructionInfo> instructions = Collections.singletonList(
 876                     new InstructionApplyActions(
 877                             Collections.singletonList(new ActionGroup(groupId))));
 878             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 879                     new InstructionApplyActions(
 880                             Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(groupId))));
 881             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 882             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 883             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 884             jobCoordinator.enqueueJob(jobKey, () -> {
 885                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 886                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 887                             NwConstants.ADD_FLOW, tx, null);
 888                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 889                             vpnName, rd)) {
 890                         optLabel.ifPresent(label -> {
 891                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 892                                 LOG.debug(
 893                                         "Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 894                                                 + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 895                                         localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 896                                         nextHopAddressList);
 897                                 makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 898                                         NwConstants.ADD_FLOW, tx);
 899                                 // If the extra-route is reachable from VMs attached to the same switch,
 900                                 // then the tunnel table can point to the load balancing group.
 901                                 // If it is reachable from VMs attached to different switches,
 902                                 // then it should be pointing to one of the local group in order to avoid looping.
 903                                 if (vrfEntry.getRoutePaths().size() == 1) {
 904                                     makeTunnelTableEntry(dpnId, label, groupId, tx);
 905                                 } else {
 906                                     makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 907                                 }
 908                             } else {
 909                                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 910                                                 + "route. LFib and Terminating table entries will not be created.",
 911                                         rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 912                             }
 913                         });
 914                     }
 915                 }));
 916             });
 917             return dpnId;
 918         }
 919         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 920                 vpnName);
 921         return BigInteger.ZERO;
 922     }
 923
 924     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 925         InstanceIdentifier<VpnToDpnList> id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId);
 926         Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 927         if (dpnInVpn.isPresent()) {
 928             return true;
 929         }
 930         return false;
 931     }
 932
 933     private LabelRouteInfo getLabelRouteInfo(Long label) {
 934         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 935             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 936         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 937         if (opResult.isPresent()) {
 938             return opResult.get();
 939         }
 940         return null;
 941     }
 942
 943     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 944         if (lri == null) {
 945             return true;
 946         }
 947
 948         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 949         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 950             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 951
 952         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 953             ? lri.getVpnInstanceList() : new ArrayList<>();
 954         if (vpnInstancesList.contains(vpnInstanceName)) {
 955             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 956             vpnInstancesList.remove(vpnInstanceName);
 957         }
 958         if (vpnInstancesList.isEmpty()) {
 959             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 960             if (tx != null) {
 961                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 962             } else {
 963                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 964             }
 965             return true;
 966         } else {
 967             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 968             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 969             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 970         }
 971         return false;
 972     }
 973
 974     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 975                                       WriteTransaction tx) {
 976         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 977
 978         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 979
 980         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 981             dpId, label, groupId);
 982     }
 983
 984     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 985                                                 WriteTransaction tx) {
 986         List<MatchInfo> mkMatches = new ArrayList<>();
 987
 988         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 989             destDpId, label, actionsInfos);
 990
 991         // Matching metadata
 992         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 993         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 994
 995         List<InstructionInfo> mkInstructions = new ArrayList<>();
 996         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 997
 998         FlowEntity terminatingServiceTableFlowEntity =
 999             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1000             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1001                 String.format("%s:%d", "TST Flow Entry ", label),
1002             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1003
1004         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1005
1006         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1007
1008         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1009         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1010             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1011             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1012             .child(Flow.class, flowKey).build();
1013         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
1014                 WriteTransaction.CREATE_MISSING_PARENTS);
1015     }
1016
1017     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1018         FlowEntity flowEntity;
1019         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1020         List<MatchInfo> mkMatches = new ArrayList<>();
1021         // Matching metadata
1022         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1023         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1024             NwConstants.INTERNAL_TUNNEL_TABLE,
1025             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1026             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1027             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1028         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
1029         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1030         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1031             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1032             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1033
1034         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1035         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1036     }
1037
1038     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1039         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1040         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1041         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1042         boolean shouldUpdateNonEcmpLocalNextHop = true;
1043         if (localNextHopInfo == null) {
1044             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1045             if (usedRds.size() > 1) {
1046                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1047                         vrfEntry.getDestPrefix(), vpnName, rd);
1048                 return returnLocalDpnId;
1049             }
1050             String vpnRd = (!usedRds.isEmpty()) ? usedRds.get(0) : rd;
1051             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1052             //in the vpn
1053             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1054                     vpnName, vpnRd, vrfEntry.getDestPrefix());
1055             if (extraRouteOptional.isPresent()) {
1056                 Routes extraRoute = extraRouteOptional.get();
1057                 String ipPrefix;
1058                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1059                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1060                 } else {
1061                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1062                 }
1063                 if (extraRoute.getNexthopIpList().size() > 1) {
1064                     shouldUpdateNonEcmpLocalNextHop = false;
1065                 }
1066                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1067                 if (localNextHopInfo != null) {
1068                     String localNextHopIP = localNextHopInfo.getIpAddress();
1069                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1070                             vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1071                     if (!dpnId.equals(BigInteger.ZERO)) {
1072                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1073                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1074                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1075                                 /*remove*/ false);
1076                         returnLocalDpnId.add(dpnId);
1077                     }
1078                 } else {
1079                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1080                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1081                 }
1082             }
1083
1084             if (localNextHopInfo == null) {
1085                 /* Imported VRF entry */
1086                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1087                 if (optionalLabel.isPresent()) {
1088                     Long label = optionalLabel.get();
1089                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1090                     LabelRouteInfo lri = getLabelRouteInfo(label);
1091                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1092                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1093                         prefixBuilder.setDpnId(lri.getDpnId());
1094                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1095                                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1096                         if (!dpnId.equals(BigInteger.ZERO)) {
1097                             returnLocalDpnId.add(dpnId);
1098                         }
1099                     }
1100                 }
1101             }
1102
1103         } else {
1104             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1105             String localNextHopIP = localNextHopInfo.getIpAddress();
1106             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1107                 vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop);
1108             if (!dpnId.equals(BigInteger.ZERO)) {
1109                 returnLocalDpnId.add(dpnId);
1110             }
1111         }
1112
1113         return returnLocalDpnId;
1114     }
1115
1116     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1117                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1118                                                 boolean shouldUpdateNonEcmpLocalNextHop) {
1119         if (localNextHopInfo != null) {
1120             final BigInteger dpnId = localNextHopInfo.getDpnId();
1121             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1122                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1123                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1124                 return dpnId;
1125             }
1126             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1127                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1128                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1129                 return dpnId;
1130             }
1131
1132             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()),
1133                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1134                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1135                             NwConstants.DEL_FLOW, tx, null);
1136                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1137                             vpnId, rd)) {
1138                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1139                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1140                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1141                                         NwConstants.DEL_FLOW, tx);
1142                                 removeTunnelTableEntry(dpnId, label, tx);
1143                             });
1144                         }
1145                     }
1146                 })));
1147             //TODO: verify below adjacency call need to be optimized (?)
1148             //In case of the removal of the extra route, the loadbalancing group is updated
1149             if (shouldUpdateNonEcmpLocalNextHop) {
1150                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1151             }
1152             return dpnId;
1153         }
1154         return BigInteger.ZERO;
1155     }
1156
1157     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1158             final VrfEntry vrfEntry, WriteTransaction tx) {
1159         if (tx == null) {
1160             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(newTx -> {
1161                 createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx);
1162             }), LOG, "Error creating remote FIB entry");
1163             return;
1164         }
1165
1166         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1167         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1168                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1169
1170         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1171         if (adjacencyResults.isEmpty()) {
1172             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1173                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1174             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1175             return;
1176         }
1177
1178         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1179         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1180                 vpnName, usedRds, vrfEntry.getDestPrefix());
1181         // create loadbalancing groups for extra routes only when the extra route is present behind
1182         // multiple VMs
1183         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1184                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1185             List<InstructionInfo> instructions = new ArrayList<>();
1186             // Obtain the local routes for this particular dpn.
1187             java.util.Optional<Routes> routes = vpnExtraRoutes
1188                     .stream()
1189                     .filter(route -> {
1190                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1191                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1192                         if (prefixToInterface == null) {
1193                             return false;
1194                         }
1195                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1196                     }).findFirst();
1197             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1198                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1199             if (groupId == FibConstants.INVALID_GROUP_ID) {
1200                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1201                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1202                 return;
1203             }
1204             List<ActionInfo> actionInfos =
1205                     Collections.singletonList(new ActionGroup(groupId));
1206             instructions.add(new InstructionApplyActions(actionInfos));
1207             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1208                     NwConstants.ADD_FLOW, tx, null);
1209         } else {
1210             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1211         }
1212
1213         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1214     }
1215
1216     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1217     /* Get interface info from prefix to interface mapping;
1218         Use the interface info to get the corresponding vpn interface op DS entry,
1219         remove the adjacency corresponding to this fib entry.
1220         If adjacency removed is the last adjacency, clean up the following:
1221          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1222          - prefix to interface entry
1223          - vpn interface op DS
1224      */
1225         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1226         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1227         if (prefixInfo == null) {
1228             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1229             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1230             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1231             if (extraRoute != null) {
1232                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1233                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1234                     if (nextHopIp != null) {
1235                         String ipPrefix;
1236                         if (isIpv4Address(nextHopIp)) {
1237                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1238                         } else {
1239                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1240                         }
1241                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1242                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1243                     }
1244                 }
1245             }
1246             if (prefixInfo == null) {
1247                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1248                 if (optionalLabel.isPresent()) {
1249                     Long label = optionalLabel.get();
1250                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1251                     LabelRouteInfo lri = getLabelRouteInfo(label);
1252                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1253                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1254                         prefixBuilder.setDpnId(lri.getDpnId());
1255                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1256                         prefixBuilder.setIpAddress(lri.getPrefix());
1257                         prefixInfo = prefixBuilder.build();
1258                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1259                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1260                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1261                     }
1262                 }
1263             }
1264         } else {
1265             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1266         }
1267     }
1268
1269     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1270                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1271
1272         if (prefixInfo == null) {
1273             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1274                     vrfEntry.getDestPrefix(), vpnId, rd);
1275             return; //Don't have any info for this prefix (shouldn't happen); need to return
1276         }
1277
1278         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1279             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1280                     vrfEntry.getDestPrefix(), vpnId, rd);
1281             return;
1282         }
1283
1284         String ifName = prefixInfo.getVpnInterfaceName();
1285         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1286             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1287     }
1288
1289     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1290         Prefixes prefixInfo;
1291         Long vpnId;
1292         String rd;
1293         VrfEntry vrfEntry;
1294         Routes extraRoute;
1295
1296         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1297                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1298             this.prefixInfo = prefixInfo;
1299             this.vpnId = vpnId;
1300             this.rd = rd;
1301             this.vrfEntry = vrfEntry;
1302             this.extraRoute = extraRoute;
1303         }
1304
1305         @Override
1306         public List<ListenableFuture<Void>> call() {
1307             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1308             // to call the respective helpers.
1309             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1310                 //First Cleanup LabelRouteInfo
1311                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1312                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1313                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1314                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1315                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1316                         synchronized (label.toString().intern()) {
1317                             LabelRouteInfo lri = getLabelRouteInfo(label);
1318                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1319                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1320                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1321                                         fibUtil.getVpnInstanceOpData(rd);
1322                                 String vpnInstanceName = "";
1323                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1324                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1325                                 }
1326                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1327                                 if (lriRemoved) {
1328                                     String parentRd = lri.getParentVpnRd();
1329                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1330                                             parentRd, vrfEntry.getDestPrefix()));
1331                                 }
1332                             } else {
1333                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1334                                         rd, vrfEntry.getDestPrefix()));
1335                             }
1336                         }
1337                     });
1338                 }
1339                 String ifName = prefixInfo.getVpnInterfaceName();
1340                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1341                 String vpnName = null;
1342
1343                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1344                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1345                     * */
1346                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1347                     if (vpnNameOpt.isPresent()) {
1348                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1349                     }
1350                 }
1351                 if (optVpnName.isPresent()) {
1352                     vpnName = optVpnName.get();
1353                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1354                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1355                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1356                     if (opVpnInterface.isPresent()) {
1357                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1358                         if (vpnId != associatedVpnId) {
1359                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1360                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1361                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1362                             return;
1363                         } else {
1364                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1365                                     vrfEntry.getDestPrefix(), associatedVpnId);
1366                         }
1367                     }
1368                 }
1369                 if (extraRoute != null) {
1370                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1371                     //Only one used Rd present in case of removal event
1372                     String usedRd = usedRds.get(0);
1373                     if (optVpnName.isPresent()) {
1374                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1375                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1376                                         vrfEntry.getDestPrefix()));
1377                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1378                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1379                     }
1380                 }
1381                 handleAdjacencyAndVpnOpInterfaceDeletion(vrfEntry, ifName, vpnName, tx);
1382             }));
1383         }
1384     }
1385
1386     /**
1387      * Check all the adjacency in VpnInterfaceOpData and decide whether to delete the entire interface or only adj.
1388      * Remove Adjacency from VPNInterfaceOpData.
1389      * if Adjacency != primary.
1390      * if Adjacency == primary , then mark it for deletion.
1391      * Remove entire VPNinterfaceOpData Entry.
1392      * if sie of Adjacency <= 2 and all are marked for deletion , delete the entire VPNinterface Op entry.
1393      * @param vrfEntry - VrfEntry removed
1394      * @param ifName - Interface name from VRFentry
1395      * @param vpnName - VPN name of corresponding VRF
1396      * @param tx - ReadWrite Tx
1397      * @throws ReadFailedException - Exception thrown in case of read failed
1398      */
1399     private void handleAdjacencyAndVpnOpInterfaceDeletion(VrfEntry vrfEntry, String ifName, String vpnName,
1400                                                           ReadWriteTransaction tx) throws ReadFailedException {
1401         InstanceIdentifier<Adjacency> adjacencyIid =
1402                 FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix());
1403         Optional<Adjacency> adjacencyOptional = tx.read(LogicalDatastoreType.OPERATIONAL, adjacencyIid).checkedGet();
1404         if (adjacencyOptional.isPresent()) {
1405             if (adjacencyOptional.get().getAdjacencyType() != Adjacency.AdjacencyType.PrimaryAdjacency) {
1406                 tx.delete(LogicalDatastoreType.OPERATIONAL,
1407                         FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1408             } else {
1409                 tx.merge(LogicalDatastoreType.OPERATIONAL, adjacencyIid,
1410                         new AdjacencyBuilder(adjacencyOptional.get()).setMarkedForDeletion(true).build());
1411             }
1412         }
1413
1414         Optional<AdjacenciesOp> optAdjacencies =
1415                 tx.read(LogicalDatastoreType.OPERATIONAL,
1416                         FibUtil.getAdjListPathOp(ifName, vpnName)).checkedGet();
1417
1418         if (!optAdjacencies.isPresent() || optAdjacencies.get().getAdjacency() == null) {
1419             return;
1420         }
1421
1422         if (optAdjacencies.get().getAdjacency().stream().count() <= 2
1423                 && optAdjacencies.get().getAdjacency().stream().allMatch(adjacency ->
1424                 adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency
1425                         && adjacency.isMarkedForDeletion() != null
1426                         && adjacency.isMarkedForDeletion()
1427         )) {
1428             LOG.info("Clean up vpn interface {} to vpn {} list.", ifName, vpnName);
1429             tx.delete(LogicalDatastoreType.OPERATIONAL,
1430                     FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1431         }
1432     }
1433
1434     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1435         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1436         final String rd = vrfTableKey.getRouteDistinguisher();
1437         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1438         if (vpnInstance == null) {
1439             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1440             return;
1441         }
1442         final Collection<VpnToDpnList> vpnToDpnList;
1443         if (vrfEntry.getParentVpnRd() != null
1444                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1445             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1446             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1447             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1448                     vpnInstance.getVpnToDpnList();
1449             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1450                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1451         } else {
1452             vpnToDpnList = vpnInstance.getVpnToDpnList();
1453         }
1454
1455         SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1456         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1457         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1458         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1459         if (subnetRoute != null) {
1460             long elanTag = subnetRoute.getElantag();
1461             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1462                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1463             if (vpnToDpnList != null) {
1464                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1465                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1466                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1467
1468                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1469                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1470                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1471                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1472                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1473                             }
1474
1475                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1476                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1477                         }
1478                     })));
1479             }
1480             optionalLabel.ifPresent(label -> {
1481                 synchronized (label.toString().intern()) {
1482                     LabelRouteInfo lri = getLabelRouteInfo(label);
1483                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1484                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1485                                 fibUtil.getVpnInstanceOpData(rd);
1486                         String vpnInstanceName = "";
1487                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1488                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1489                         }
1490                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1491                         if (lriRemoved) {
1492                             String parentRd = lri.getParentVpnRd();
1493                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1494                                     parentRd, vrfEntry.getDestPrefix()));
1495                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1496                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1497                         }
1498                     } else {
1499                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1500                                 rd, vrfEntry.getDestPrefix()));
1501                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1502                                 label, rd, vrfEntry.getDestPrefix());
1503                     }
1504                 }
1505             });
1506             return;
1507         }
1508
1509         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1510             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1511         if (vpnToDpnList != null) {
1512             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1513                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1514             String jobKey;
1515             Optional<Routes> extraRouteOptional;
1516             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1517             if (usedRds != null && !usedRds.isEmpty()) {
1518                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1519                 if (usedRds.size() > 1) {
1520                     LOG.error("The extra route prefix is still present in some DPNs");
1521                     return ;
1522                 } else {
1523                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1524                     //is not present in any other DPN
1525                     extraRouteOptional = VpnExtraRouteHelper
1526                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1527                 }
1528             } else {
1529                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1530                 extraRouteOptional = Optional.absent();
1531             }
1532
1533             jobCoordinator.enqueueJob(jobKey,
1534                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1535                     if (localDpnIdList.size() <= 0) {
1536                         for (VpnToDpnList curDpn : vpnToDpnList) {
1537                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1538                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1539                         }
1540                     } else {
1541                         for (BigInteger localDpnId : localDpnIdList) {
1542                             for (VpnToDpnList curDpn : vpnToDpnList) {
1543                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1544                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1545                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1546                                 }
1547                             }
1548                         }
1549                     }
1550                 })), MAX_RETRIES);
1551         }
1552
1553         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1554         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1555         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1556
1557         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1558         // of the interVpnLink.
1559         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1560         if (optVpnUuid.isPresent()) {
1561             String vpnUuid = optVpnUuid.get();
1562             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1563                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1564                 if (optInterVpnLink.isPresent()) {
1565                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1566                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1567                         // This is route that points to the other endpoint of an InterVpnLink
1568                         // In that case, we should look for the FIB table pointing to
1569                         // LPortDispatcher table and remove it.
1570                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1571                     }
1572                 }
1573             });
1574         }
1575
1576     }
1577
1578     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1579                                     int addOrRemove, WriteTransaction tx) {
1580         if (tx == null) {
1581             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1582                 newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG,
1583                 "Error making LFIB table entry");
1584             return;
1585         }
1586
1587         List<MatchInfo> matches = new ArrayList<>();
1588         matches.add(MatchEthernetType.MPLS_UNICAST);
1589         matches.add(new MatchMplsLabel(label));
1590
1591         // Install the flow entry in L3_LFIB_TABLE
1592         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1593
1594         FlowEntity flowEntity;
1595         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1596             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1597         Flow flow = flowEntity.getFlowBuilder().build();
1598         String flowId = flowEntity.getFlowId();
1599         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1600         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1601         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1602             .child(Node.class, nodeDpn.key()).augmentation(FlowCapableNode.class)
1603             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1604
1605         if (addOrRemove == NwConstants.ADD_FLOW) {
1606             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1607         } else {
1608             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1609         }
1610
1611         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1612             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1613     }
1614
1615     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1616                                     final FutureCallback<List<Void>> callback) {
1617         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1618         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1619             () -> {
1620                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1621                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1622                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1623                         LogicalDatastoreType.CONFIGURATION, id);
1624                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1625                 if (!vrfTable.isPresent()) {
1626                     LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1627                     if (callback != null) {
1628                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1629                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1630                     }
1631                     return futures;
1632                 }
1633                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1634                     futures.add(retryingTxRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
1635                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1636                             SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1637                             if (subnetRoute != null) {
1638                                 long elanTag = subnetRoute.getElantag();
1639                                 installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1640                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW,
1641                                         tx);
1642                                 continue;
1643                             }
1644                             RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1645                             if (routerInt != null) {
1646                                 LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1647                                         rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1648                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1649                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1650                                         NwConstants.ADD_FLOW);
1651                                 continue;
1652                             }
1653                             //Handle local flow creation for imports
1654                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1655                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1656                                 if (optionalLabel.isPresent()) {
1657                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1658                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1659                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1660                                         if (lri.getDpnId().equals(dpnId)) {
1661                                             try {
1662                                                 int etherType = NWUtil.getEtherTypeFromIpPrefix(
1663                                                         vrfEntry.getDestPrefix());
1664                                                 createLocalFibEntry(vpnId, rd, vrfEntry, etherType);
1665                                             } catch (IllegalArgumentException ex) {
1666                                                 LOG.warn("Unable to get etherType for IP Prefix {}",
1667                                                         vrfEntry.getDestPrefix());
1668                                             }
1669                                             continue;
1670                                         }
1671                                     }
1672                                 }
1673                             }
1674                             boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1675                                     vrfEntry, dpnId);
1676                             if (shouldCreateRemoteFibEntry) {
1677                                 LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId);
1678                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1679                                     List<SubTransaction> txnObjects =  new ArrayList<>();
1680                                     bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1681                                             vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1682                                 } else {
1683                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1684                                             vrfEntry, tx);
1685                                 }
1686                             }
1687                         }
1688                     }));
1689                     if (callback != null) {
1690                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1691                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1692                     }
1693                 }
1694                 return futures;
1695             });
1696     }
1697
1698     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1699                                             final String localNextHopIp, final String remoteNextHopIp) {
1700         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1701             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1702         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1703         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1704         List<SubTransaction> txnObjects =  new ArrayList<>();
1705         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1706         if (vrfTable.isPresent()) {
1707             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1708                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1709                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1710                         vrfTable.get().getVrfEntry().stream()
1711                                 .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1712                                 .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1713                                         rd, remoteNextHopIp, vrfTable, tx, txnObjects));
1714                     }
1715                 })));
1716         }
1717     }
1718
1719     public void manageRemoteRouteOnDPN(final boolean action,
1720                                        final BigInteger localDpnId,
1721                                        final long vpnId,
1722                                        final String rd,
1723                                        final String destPrefix,
1724                                        final String destTepIp,
1725                                        final long label) {
1726         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1727
1728         if (vpnInstance == null) {
1729             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1730             return;
1731         }
1732
1733         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1734             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1735                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1736                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1737                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1738                     if (vrfEntry == null) {
1739                         return;
1740                     }
1741                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1742                             action, localDpnId, vpnId, rd, destPrefix);
1743                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1744                     VrfEntry modVrfEntry;
1745                     if (routePathList == null || routePathList.isEmpty()) {
1746                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1747                                 Collections.singletonList(destTepIp),
1748                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1749                     } else {
1750                         modVrfEntry = vrfEntry;
1751                     }
1752
1753                     if (action) {
1754                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1755                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1756                                 modVrfEntry, tx);
1757                     } else {
1758                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1759                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1760                                 vrfEntry.getDestPrefix());
1761                         if (usedRds.size() > 1) {
1762                             LOG.debug("The extra route prefix is still present in some DPNs");
1763                             return;
1764                         }
1765                         //Is this fib route an extra route? If yes, get the nexthop which would be
1766                         //an adjacency in the vpn
1767                         Optional<Routes> extraRouteOptional = Optional.absent();
1768                         if (usedRds.size() != 0) {
1769                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1770                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1771                                     usedRds.get(0), vrfEntry.getDestPrefix());
1772                         }
1773                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1774                                 extraRouteOptional, tx);
1775                     }
1776                 }
1777             })));
1778     }
1779
1780     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1781                                  final FutureCallback<List<Void>> callback) {
1782         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1783         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1784             () -> {
1785                 InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1786                 final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1787                 List<SubTransaction> txnObjects =  new ArrayList<>();
1788                 final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker,
1789                         LogicalDatastoreType.CONFIGURATION, id);
1790                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1791                 if (vrfTable.isPresent()) {
1792                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1793                         futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1794                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1795                             for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1796                                 /* Handle subnet routes here */
1797                                 SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class);
1798                                 if (subnetRoute != null && !fibUtil
1799                                         .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1800                                     LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1801                                             + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1802                                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1803                                             NwConstants.DEL_FLOW, tx, null);
1804                                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1805                                     if (routePaths != null) {
1806                                         for (RoutePaths routePath : routePaths) {
1807                                             makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1808                                                     DEFAULT_FIB_FLOW_PRIORITY,
1809                                                     NwConstants.DEL_FLOW, tx);
1810                                             LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}"
1811                                                     + " for rd {} prefix {}", routePath.getLabel(), rd,
1812                                                     vrfEntry.getDestPrefix());
1813                                         }
1814                                     }
1815                                     installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1816                                             NwConstants.DEL_FLOW, tx);
1817                                     continue;
1818                                 }
1819                                 // ping responder for router interfaces
1820                                 RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class);
1821                                 if (routerInt != null) {
1822                                     LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1823                                             rd, routerInt.getUuid(), routerInt.getIpAddress(),
1824                                             routerInt.getMacAddress());
1825                                     routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1826                                             routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1827                                             NwConstants.DEL_FLOW);
1828                                     continue;
1829                                 }
1830
1831                                 //Handle local flow deletion for imports
1832                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1833                                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1834                                     if (optionalLabel.isPresent()) {
1835                                         List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1836                                         LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1837                                         if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList,
1838                                                 lri) && lri.getDpnId().equals(dpnId)) {
1839                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1840                                         }
1841                                     }
1842                                 }
1843
1844                                 // Passing null as we don't know the dpn
1845                                 // to which prefix is attached at this point
1846                                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1847                                         vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1848                                 Optional<Routes> extraRouteOptional;
1849                                 //Is this fib route an extra route? If yes, get the nexthop which would be
1850                                 //an adjacency in the vpn
1851                                 if (usedRds != null && !usedRds.isEmpty()) {
1852                                     if (usedRds.size() > 1) {
1853                                         LOG.error("The extra route prefix is still present in some DPNs");
1854                                         return;
1855                                     } else {
1856                                         extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1857                                                 usedRds.get(0), vrfEntry.getDestPrefix());
1858
1859                                     }
1860                                 } else {
1861                                     extraRouteOptional = Optional.absent();
1862                                 }
1863                                 if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1864                                     bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1865                                             vrfTable.get().key(), vrfEntry, extraRouteOptional, tx, txnObjects);
1866                                 } else {
1867                                     if (subnetRoute == null || !fibUtil
1868                                             .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) {
1869                                         baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId,
1870                                                 vrfTable.get().key(), vrfEntry, extraRouteOptional, tx);
1871                                     }
1872                                 }
1873                             }
1874                         }));
1875                     }
1876                     if (callback != null) {
1877                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1878                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1879                     }
1880                 } else {
1881                     LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId);
1882                 }
1883                 return futures;
1884             });
1885
1886     }
1887
1888     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1889                                            final String localNextHopIp, final String remoteNextHopIp) {
1890         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1891                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1892             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1893         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1894         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1895         List<SubTransaction> txnObjects =  new ArrayList<>();
1896         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1897         if (vrfTable.isPresent()) {
1898             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1899                 () -> {
1900                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1901                         return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1902                             tx -> vrfTable.get().getVrfEntry().stream()
1903                                     .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1904                                     .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId,
1905                                             remoteNextHopIp, vrfTable, tx, txnObjects))));
1906                     }
1907                 });
1908         }
1909     }
1910
1911     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1912         InstanceIdentifierBuilder<VrfTables> idBuilder =
1913             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1914         return idBuilder.build();
1915     }
1916
1917     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1918         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1919                 .FLOWID_SEPARATOR + nextHop;
1920     }
1921
1922     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1923         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1924                 + tableMiss + FLOWID_PREFIX;
1925     }
1926
1927     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1928         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1929             .child(VrfTables.class, new VrfTablesKey(rd))
1930             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1931         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1932         if (vrfEntry.isPresent()) {
1933             return vrfEntry.get();
1934         }
1935         return null;
1936     }
1937
1938     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1939                                              final String vpnName,
1940                                              final VrfEntry vrfEntry) {
1941         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1942
1943         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1944         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1945
1946         if (targetDpns.isEmpty()) {
1947             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1948             return;
1949         }
1950
1951         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1952         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1953
1954         // delete from FIB
1955         //
1956         optNextHop.ifPresent(nextHop -> {
1957             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1958             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1959             Flow flow = new FlowBuilder().withKey(flowKey).setId(new FlowId(flowRef))
1960                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1961
1962             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1963             for (BigInteger dpId : targetDpns) {
1964                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1965                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1966
1967                 mdsalManager.removeFlow(dpId, flow);
1968             }
1969         });
1970
1971         // delete from LFIB
1972         //
1973         optLabel.ifPresent(label -> {
1974             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1975
1976             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1977                 for (BigInteger dpId : targetDpns) {
1978                     LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1979                             vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1980                     makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW,
1981                             tx);
1982                 }
1983             }), LOG, "Error removing flows");
1984         });
1985     }
1986
1987     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1988             List<String> nextHopAddressList, LabelRouteInfo lri) {
1989         return lri != null && lri.getPrefix().equals(prefix)
1990                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1991     }
1992
1993     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1994         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1995             return true;
1996         }
1997
1998         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1999         if (prefix != null) {
2000             BigInteger prefixDpnId = prefix.getDpnId();
2001             if (dpnId.equals(prefixDpnId)) {
2002                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2003                         vrfEntry, dpnId);
2004                 return false;
2005             }
2006         }
2007         return true;
2008     }
2009 }