fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.CheckedFuture;
  15 import com.google.common.util.concurrent.FutureCallback;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  20 import java.math.BigInteger;
  21 import java.net.InetAddress;
  22 import java.net.UnknownHostException;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Collections;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.CopyOnWriteArrayList;
  30 import java.util.concurrent.ExecutionException;
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.mdsalutil.ActionInfo;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  44 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  45 import org.opendaylight.genius.mdsalutil.MatchInfo;
  46 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  51 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  52 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  53 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  54 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  55 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  56 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  59 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  60 import org.opendaylight.genius.utils.ServiceIndex;
  61 import org.opendaylight.genius.utils.batching.SubTransaction;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  64 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  67 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  68 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  69 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 101 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 102 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 103 import org.slf4j.Logger;
 104 import org.slf4j.LoggerFactory;
 105
 106
 107 @Singleton
 108 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> {
 109
 110     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 111     private static final String FLOWID_PREFIX = "L3.";
 112     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 113     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 114     private static final int IPV4_ADDR_PREFIX_LENGTH = 32;
 115     private static final int LFIB_INTERVPN_PRIORITY = 15;
 116     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 117     private static final int DJC_MAX_RETRIES = 3;
 118     private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 119
 120     private final DataBroker dataBroker;
 121     private final ManagedNewTransactionRunner txRunner;
 122     private final IMdsalApiManager mdsalManager;
 123     private final NexthopManager nextHopManager;
 124     private final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler;
 125     private final BaseVrfEntryHandler baseVrfEntryHandler;
 126     private final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler;
 127     private final JobCoordinator jobCoordinator;
 128     private final IElanService elanManager;
 129     private final FibUtil fibUtil;
 130     private final InterVpnLinkCache interVpnLinkCache;
 131     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
 132
 133     @Inject
 134     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 135                             final NexthopManager nexthopManager,
 136                             final IElanService elanManager,
 137                             final BaseVrfEntryHandler vrfEntryHandler,
 138                             final BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler,
 139                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
 140                             final JobCoordinator jobCoordinator,
 141                             final FibUtil fibUtil,
 142                             final InterVpnLinkCache interVpnLinkCache) {
 143         super(VrfEntry.class, VrfEntryListener.class);
 144         this.dataBroker = dataBroker;
 145         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 146         this.mdsalManager = mdsalApiManager;
 147         this.nextHopManager = nexthopManager;
 148         this.elanManager = elanManager;
 149         this.baseVrfEntryHandler = vrfEntryHandler;
 150         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;
 151         this.routerInterfaceVrfEntryHandler = routerInterfaceVrfEntryHandler;
 152         this.jobCoordinator = jobCoordinator;
 153         this.fibUtil = fibUtil;
 154         this.interVpnLinkCache = interVpnLinkCache;
 155     }
 156
 157     @Override
 158     @PostConstruct
 159     public void init() {
 160         LOG.info("{} init", getClass().getSimpleName());
 161         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 162     }
 163
 164     @Override
 165     @SuppressWarnings("checkstyle:IllegalCatch")
 166     public void close() {
 167         closeables.forEach(c -> {
 168             try {
 169                 c.close();
 170             } catch (Exception e) {
 171                 LOG.warn("Error closing {}", c, e);
 172             }
 173         });
 174     }
 175
 176     @Override
 177     protected VrfEntryListener getDataTreeChangeListener() {
 178         return VrfEntryListener.this;
 179     }
 180
 181     @Override
 182     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 183         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 184     }
 185
 186     @Override
 187     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 188         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 189         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 190         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 191                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 192         addFibEntries(identifier, vrfEntry, rd);
 193         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 194                  rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 195     }
 196
 197     //This method is temporary. Eventually Factory design pattern will be used to get
 198     // right VrfEntryhandle and invoke its methods.
 199     private void addFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 200         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 201             bgpRouteVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 202             return;
 203         }
 204         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 205             LOG.info("EVPN flows need to be programmed.");
 206             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 207                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 208             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 209             closeables.add(evpnVrfEntryHandler);
 210             return;
 211         }
 212         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
 213         if (routerInt != null) {
 214             // ping responder for router interfaces
 215             routerInterfaceVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
 216             return;
 217         }
 218         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 219             createFibEntries(identifier, vrfEntry);
 220             return;
 221         }
 222     }
 223
 224     @Override
 225     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 226         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 227         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 228         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 229                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 230         removeFibEntries(identifier, vrfEntry, rd);
 231         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 232             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 233     }
 234
 235     //This method is temporary. Eventually Factory design pattern will be used to get
 236     // right VrfEntryhandle and invoke its methods.
 237     private void removeFibEntries(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
 238         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 239             LOG.info("EVPN flows to be deleted");
 240             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
 241                     nextHopManager, jobCoordinator, elanManager, fibUtil);
 242             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 243             closeables.add(evpnVrfEntryHandler);
 244             return;
 245         }
 246         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
 247         if (routerInt != null) {
 248             // ping responder for router interfaces
 249             routerInterfaceVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 250             return;
 251         }
 252         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 253             deleteFibEntries(identifier, vrfEntry);
 254             return;
 255         }
 256         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 257             bgpRouteVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
 258             return;
 259         }
 260     }
 261
 262     @Override
 263     // "Redundant nullcheck of originalRoutePath, which is known to be non-null" - the null checking for
 264     // originalRoutePath is a little dicey - safest to keep the checking even if not needed.
 265     @SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
 266     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 267         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 268         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 269         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd,
 270                 update.getDestPrefix(), update.getRoutePaths(), update.getOrigin(), original.getOrigin());
 271         // Handle BGP Routes first
 272         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 273             bgpRouteVrfEntryHandler.updateFlows(identifier, original, update, rd);
 274             LOG.info("UPDATE: Updated BGP advertised Fib Entry with rd {} prefix {} route-paths {}",
 275                     rd, update.getDestPrefix(), update.getRoutePaths());
 276             return;
 277         }
 278
 279         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.STATIC) {
 280             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 281             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 282             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 283
 284             //Updates need to be handled for extraroute even if original vrf entry route path is null or
 285             //updated vrf entry route path is null. This can happen during tunnel events.
 286             Optional<VpnInstanceOpDataEntry> optVpnInstance = fibUtil.getVpnInstanceOpData(rd);
 287             List<String> usedRds = new ArrayList<>();
 288             if (optVpnInstance.isPresent()) {
 289                 usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,optVpnInstance.get().getVpnId(),
 290                         update.getDestPrefix());
 291             }
 292             // If original VRF Entry had nexthop null , but update VRF Entry
 293             // has nexthop , route needs to be created on remote Dpns
 294             if (originalRoutePath == null || originalRoutePath.isEmpty()
 295                     && updateRoutePath != null && !updateRoutePath.isEmpty() && usedRds.isEmpty()) {
 296                 // TODO(vivek): Though ugly, Not handling this code now, as each
 297                 // tep add event will invoke flow addition
 298                 LOG.trace("Original VRF entry NH is null for destprefix {}. And the prefix is not an extra route."
 299                         + " This event is IGNORED here.", update.getDestPrefix());
 300                 return;
 301             }
 302
 303             // If original VRF Entry had valid nexthop , but update VRF Entry
 304             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 305             if (updateRoutePath == null || updateRoutePath.isEmpty()
 306                     && originalRoutePath != null && !originalRoutePath.isEmpty() && usedRds.isEmpty()) {
 307                 LOG.trace("Original VRF entry had valid NH for destprefix {}. And the prefix is not an extra route."
 308                         + "This event is IGNORED here.", update.getDestPrefix());
 309                 return;
 310             }
 311             //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
 312             List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
 313             nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
 314             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 315             nextHopsRemoved.parallelStream()
 316                     .forEach(nextHopRemoved -> fibUtil.updateUsedRdAndVpnToExtraRoute(
 317                             writeOperTxn, nextHopRemoved, rd, update.getDestPrefix()));
 318             CheckedFuture<Void, TransactionCommitFailedException> operFuture = writeOperTxn.submit();
 319             try {
 320                 operFuture.get();
 321             } catch (InterruptedException | ExecutionException e) {
 322                 LOG.error("Exception encountered while submitting operational future for update vrfentry {}: "
 323                         + "{}", update, e);
 324             }
 325
 326             createFibEntries(identifier, update);
 327             LOG.info("UPDATE: Updated static Fib Entry with rd {} prefix {} route-paths {}",
 328                     rd, update.getDestPrefix(), update.getRoutePaths());
 329             return;
 330         }
 331
 332         //Handle all other routes only on a cluster reboot
 333         if (original.equals(update)) {
 334             //Reboot use-case
 335             createFibEntries(identifier, update);
 336             LOG.info("UPDATE: Updated Non-static Fib Entry with rd {} prefix {} route-paths {}",
 337                     rd, update.getDestPrefix(), update.getRoutePaths());
 338             return;
 339         }
 340
 341         LOG.info("UPDATE: Ignoring update for FIB entry with rd {} prefix {} route-origin {} route-paths {}",
 342                 rd, update.getDestPrefix(), update.getOrigin(), update.getRoutePaths());
 343     }
 344
 345     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 346         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 347         List<SubTransaction> txnObjects =  new ArrayList<>();
 348         final VpnInstanceOpDataEntry vpnInstance =
 349                 fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
 350         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 351         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 352                 + " has null vpnId!");
 353         final Collection<VpnToDpnList> vpnToDpnList;
 354         if (vrfEntry.getParentVpnRd() != null
 355                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 356             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
 357             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
 358             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 359                 vpnInstance.getVpnToDpnList();
 360             LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}",
 361                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
 362         } else {
 363             vpnToDpnList = vpnInstance.getVpnToDpnList();
 364         }
 365         final Long vpnId = vpnInstance.getVpnId();
 366         final String rd = vrfTableKey.getRouteDistinguisher();
 367         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 368         if (subnetRoute != null) {
 369             final long elanTag = subnetRoute.getElantag();
 370             LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
 371                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
 372             if (vpnToDpnList != null) {
 373                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> {
 374                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 375                     for (final VpnToDpnList curDpn : vpnToDpnList) {
 376                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 377                             installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 378                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(),
 379                                     vrfEntry, NwConstants.ADD_FLOW, tx);
 380                         }
 381                     }
 382                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 383                     futures.add(tx.submit());
 384                     return futures;
 385                 });
 386             }
 387             return;
 388         }
 389
 390         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 391         if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
 392             jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> {
 393                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 394                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
 395                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 396                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 397                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 398                                 try {
 399                                     if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
 400                                         bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(),
 401                                                 vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx, txnObjects);
 402                                     } else {
 403                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
 404                                                 vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
 405                                     }
 406                                 } catch (NullPointerException e) {
 407                                     LOG.error("Failed to get create remote fib flows for prefix {} ",
 408                                             vrfEntry.getDestPrefix(), e);
 409                                 }
 410                             }
 411                         }
 412                     }
 413                 }
 414                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 415                 futures.add(tx.submit());
 416                 return futures;
 417             }, DJC_MAX_RETRIES);
 418         }
 419
 420         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
 421         if (optVpnUuid.isPresent()) {
 422             String vpnUuid = optVpnUuid.get();
 423             InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
 424             if (interVpnLink != null) {
 425                 LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
 426                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 427                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 428                         // This is an static route that points to the other endpoint of an InterVpnLink
 429                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 430                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 431                         installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
 432                     }
 433                 });
 434             }
 435         }
 436     }
 437
 438     void refreshFibTables(String rd, String prefix) {
 439         InstanceIdentifier<VrfEntry> vrfEntryId =
 440                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
 441                         .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
 442         Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
 443         if (vrfEntry.isPresent()) {
 444             createFibEntries(vrfEntryId, vrfEntry.get());
 445         }
 446     }
 447
 448     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 449         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 450         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 451         prefixBuilder.setDpnId(lri.getDpnId());
 452         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 453         prefixBuilder.setIpAddress(lri.getPrefix());
 454         // Increment the refCount here
 455         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 456             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 457         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 458         if (!isPresentInList) {
 459             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 460             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 461             vpnInstanceNames.add(vpnInstanceName);
 462             builder.setVpnInstanceList(vpnInstanceNames);
 463             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 464         } else {
 465             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 466         }
 467         return prefixBuilder.build();
 468     }
 469
 470     void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 471                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 472         Boolean wrTxPresent = true;
 473         if (tx == null) {
 474             wrTxPresent = false;
 475             tx = dataBroker.newWriteOnlyTransaction();
 476         }
 477         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 478             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 479             synchronized (label.toString().intern()) {
 480                 LabelRouteInfo lri = getLabelRouteInfo(label);
 481                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 482
 483                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 484                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 485                                 fibUtil.getVpnInstanceOpData(rd);
 486                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 487                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 488                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 489                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 490                             }
 491                         }
 492                     }
 493                     LOG.debug("SUBNETROUTE: installSubnetRouteInFib: Fetched labelRouteInfo for label {} interface {}"
 494                             + " and got dpn {}", label, lri.getVpnInterfaceName(), lri.getDpnId());
 495                 }
 496             }
 497         });
 498         final List<InstructionInfo> instructions = new ArrayList<>();
 499         BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24)
 500             .or(BigInteger.valueOf(vpnId).shiftLeft(1));
 501         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 502         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 503         baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 504                 NwConstants.ADD_FLOW, tx, null);
 505
 506         if (vrfEntry.getRoutePaths() != null) {
 507             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 508                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 509                     List<ActionInfo> actionsInfos = new ArrayList<>();
 510                     // reinitialize instructions list for LFIB Table
 511                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 512
 513                     actionsInfos.add(new ActionPopMpls());
 514                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 515                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 516                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 517                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 518
 519                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 520                             NwConstants.ADD_FLOW, tx);
 521                 }
 522             }
 523         }
 524         if (!wrTxPresent) {
 525             tx.submit();
 526         }
 527     }
 528
 529     private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId,
 530                                                     final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
 531         List<MatchInfo> matches = new ArrayList<>();
 532
 533         LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}",
 534                 vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
 535         String[] ipAddress = vrfEntry.getDestPrefix().split("/");
 536         String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
 537         final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
 538
 539         InetAddress destPrefix;
 540         try {
 541             destPrefix = InetAddress.getByName(subnetBroadcastAddr);
 542         } catch (UnknownHostException e) {
 543             LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}",
 544                     vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
 545             return;
 546         }
 547
 548         // Match on VpnId and SubnetBroadCast IP address
 549         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 550         matches.add(MatchEthernetType.IPV4);
 551
 552         if (prefixLength != 0) {
 553             matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
 554         }
 555
 556         //Action is to drop the packet
 557         List<InstructionInfo> dropInstructions = new ArrayList<>();
 558         List<ActionInfo> actionsInfos = new ArrayList<>();
 559         actionsInfos.add(new ActionDrop());
 560         dropInstructions.add(new InstructionApplyActions(actionsInfos));
 561
 562         int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
 563         String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
 564         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority,
 565                 flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
 566
 567         Flow flow = flowEntity.getFlowBuilder().build();
 568         String flowId = flowEntity.getFlowId();
 569         FlowKey flowKey = new FlowKey(new FlowId(flowId));
 570         Node nodeDpn = FibUtil.buildDpnNode(dpnId);
 571
 572         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 573                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 574                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 575
 576         if (addOrRemove == NwConstants.ADD_FLOW) {
 577             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
 578         } else {
 579             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 580         }
 581     }
 582
 583     /*
 584      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 585      * LportDispatcher table (via table 80)
 586      */
 587     private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
 588                                             final VrfEntry vrfEntry) {
 589         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 590         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 591         // packet is commuted from Vpn2 to Vpn1.
 592         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 593         if (!interVpnLink.isActive()) {
 594             LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
 595                      interVpnLinkName, vrfEntry.getDestPrefix());
 596             return;
 597         }
 598
 599         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
 600         Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
 601         if (!optLportTag.isPresent()) {
 602             LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
 603             return;
 604         }
 605
 606         Long lportTag = optLportTag.get();
 607         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
 608         if (label == null) {
 609             LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
 610                       vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 611             return;
 612         }
 613         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 614         List<InstructionInfo> instructions = Arrays.asList(
 615             new InstructionApplyActions(actionsInfos),
 616             new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 617                                                             ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 618                                                                                   NwConstants.L3VPN_SERVICE_INDEX)),
 619                                          MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 620             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 621         List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 622
 623         for (BigInteger dpId : targetDpns) {
 624             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 625                       vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 626
 627             makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
 628                                /*writeTx*/null);
 629         }
 630     }
 631
 632
 633     /*
 634      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 635      */
 636     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 637                                                final VrfEntry vrfEntry, long vpnTag) {
 638         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 639         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 640             && vrfEntry.getRoutePaths().size() == 1);
 641         String destination = vrfEntry.getDestPrefix();
 642         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 643         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 644
 645         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 646         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 647         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 648         if (interVpnLink.getState().or(State.Error) != State.Active) {
 649             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 650                 destination, nextHop, interVpnLinkName);
 651             return;
 652         }
 653
 654         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 655         if (!optOtherEndpointLportTag.isPresent()) {
 656             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 657                 vpnUuid, interVpnLinkName);
 658             return;
 659         }
 660
 661         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 662         if (targetDpns.isEmpty()) {
 663             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 664                 vpnUuid, interVpnLinkName);
 665             return;
 666         }
 667
 668         String[] values = destination.split("/");
 669         String destPrefixIpAddress = values[0];
 670         int prefixLength = values.length == 1 ? 0 : Integer.parseInt(values[1]);
 671
 672         List<MatchInfo> matches = new ArrayList<>();
 673         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 674         matches.add(MatchEthernetType.IPV4);
 675
 676         if (prefixLength != 0) {
 677             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 678         }
 679
 680         List<Instruction> instructions =
 681             Arrays.asList(new InstructionWriteMetadata(
 682                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 683                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 684                             .L3VPN_SERVICE_INDEX)),
 685                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 686                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 687
 688         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 689         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 690         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 691             COOKIE_VM_FIB_TABLE, matches, instructions);
 692
 693         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 694             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 695
 696         for (BigInteger dpId : targetDpns) {
 697
 698             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 699                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 700                 dpId, interVpnLink.getInterVpnLinkName());
 701
 702             mdsalManager.installFlow(dpId, flowEntity);
 703         }
 704     }
 705
 706     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 707         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 708         String localNextHopIP = vrfEntry.getDestPrefix();
 709         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 710         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 711         if (localNextHopInfo == null) {
 712             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP);
 713             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 714                     vpnName, usedRds, localNextHopIP);
 715             boolean localNextHopSeen = false;
 716             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 717             for (Routes vpnExtraRoute : vpnExtraRoutes) {
 718                 String ipPrefix;
 719                 if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) {
 720                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 721                 } else {
 722                     ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 723                 }
 724                 Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId,
 725                     ipPrefix);
 726                 if (localNextHopInfoLocal != null) {
 727                     localNextHopSeen = true;
 728                     BigInteger dpnId =
 729                             checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
 730                                     vpnId, rd, vrfEntry, vpnId, vpnExtraRoute, vpnExtraRoutes);
 731                     returnLocalDpnId.add(dpnId);
 732                 }
 733             }
 734             if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 735                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 736                 if (optionalLabel.isPresent()) {
 737                     Long label = optionalLabel.get();
 738                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 739                     synchronized (label.toString().intern()) {
 740                         LabelRouteInfo lri = getLabelRouteInfo(label);
 741                         if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) {
 742                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 743                                     fibUtil.getVpnInstanceOpData(rd);
 744                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 745                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 746                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 747                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 748                                     localNextHopIP = lri.getPrefix();
 749                                 } else {
 750                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 751                                     localNextHopIP = lri.getPrefix();
 752                                 }
 753                             }
 754                             if (localNextHopInfo != null) {
 755                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 756                                         label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 757                                 if (vpnExtraRoutes.isEmpty()) {
 758                                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 759                                             vpnId, rd, vrfEntry, lri.getParentVpnid(), null, vpnExtraRoutes);
 760                                     returnLocalDpnId.add(dpnId);
 761                                 } else {
 762                                     for (Routes extraRoutes : vpnExtraRoutes) {
 763                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo,
 764                                                 localNextHopIP,
 765                                                 vpnId, rd, vrfEntry, lri.getParentVpnid(),
 766                                                 extraRoutes, vpnExtraRoutes);
 767                                         returnLocalDpnId.add(dpnId);
 768                                     }
 769                                 }
 770                             }
 771                         }
 772                     }
 773                 }
 774             }
 775             if (returnLocalDpnId.isEmpty()) {
 776                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 777             }
 778         } else {
 779             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 780                     rd, vrfEntry, vpnId, /*routes*/ null, /*vpnExtraRoutes*/ null);
 781             if (dpnId != null) {
 782                 returnLocalDpnId.add(dpnId);
 783             }
 784         }
 785         return returnLocalDpnId;
 786     }
 787
 788     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 789                                                 final Long vpnId, final String rd,
 790                                                 final VrfEntry vrfEntry, Long parentVpnId,
 791                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 792         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 793         if (localNextHopInfo != null) {
 794             long groupId;
 795             long localGroupId;
 796             final BigInteger dpnId = localNextHopInfo.getDpnId();
 797             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
 798                 LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 799                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 800                 return dpnId;
 801             }
 802             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
 803                 LOG.debug("checkCreateLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
 804                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 805                 return dpnId;
 806             }
 807             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 808             String interfaceName = localNextHopInfo.getVpnInterfaceName();
 809             String prefix = vrfEntry.getDestPrefix();
 810             String gwMacAddress = vrfEntry.getGatewayMacAddress();
 811             //The loadbalancing group is created only if the extra route has multiple nexthops
 812             //to avoid loadbalancing the discovered routes
 813             if (vpnExtraRoutes != null) {
 814                 if (isIpv4Address(routes.getNexthopIpList().get(0))) {
 815                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
 816                 } else {
 817                     localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
 818                 }
 819                 if (vpnExtraRoutes.size() > 1) {
 820                     groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 821                             vpnExtraRoutes);
 822                     localGroupId = nextHopManager.getLocalNextHopGroup(parentVpnId, localNextHopIP);
 823                 } else if (routes.getNexthopIpList().size() > 1) {
 824                     groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 825                             vpnExtraRoutes);
 826                     localGroupId = groupId;
 827                 } else {
 828                     groupId = nextHopManager.getLocalNextHopGroup(parentVpnId, localNextHopIP);
 829                     localGroupId = groupId;
 830                 }
 831             } else {
 832                 groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, interfaceName, localNextHopIP, prefix,
 833                         gwMacAddress, jobKey);
 834                 localGroupId = groupId;
 835             }
 836             if (groupId == FibConstants.INVALID_GROUP_ID) {
 837                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 838                         prefix, rd, interfaceName, dpnId.toString());
 839                 return BigInteger.ZERO;
 840             }
 841             final List<InstructionInfo> instructions = Collections.singletonList(
 842                     new InstructionApplyActions(
 843                             Collections.singletonList(new ActionGroup(groupId))));
 844             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 845                     new InstructionApplyActions(
 846                             Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 847             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 848             List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
 849             jobCoordinator.enqueueJob(jobKey, () -> {
 850                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 851                 baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions,
 852                         NwConstants.ADD_FLOW, tx, null);
 853                 if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
 854                         vpnName, rd)) {
 855                     optLabel.ifPresent(label -> {
 856                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 857                             LOG.debug("Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
 858                                             + "{}, rd {}, prefix {}, nexthop {}", dpnId,
 859                                     localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
 860                                     nextHopAddressList);
 861                             makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 862                                     NwConstants.ADD_FLOW, tx);
 863                             // If the extra-route is reachable from VMs attached to the same switch,
 864                             // then the tunnel table can point to the load balancing group.
 865                             // If it is reachable from VMs attached to different switches,
 866                             // then it should be pointing to one of the local group in order to avoid looping.
 867                             if (vrfEntry.getRoutePaths().size() == 1) {
 868                                 makeTunnelTableEntry(dpnId, label, groupId, tx);
 869                             } else {
 870                                 makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 871                             }
 872                         } else {
 873                             LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported "
 874                                             + "route. LFib and Terminating table entries will not be created.",
 875                                     rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 876                         }
 877                     });
 878                 }
 879                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 880                 futures.add(tx.submit());
 881                 return futures;
 882             });
 883             return dpnId;
 884         }
 885         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}", vrfEntry.getDestPrefix(), rd,
 886                 vpnName);
 887         return BigInteger.ZERO;
 888     }
 889
 890     private LabelRouteInfo getLabelRouteInfo(Long label) {
 891         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 892             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 893         Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 894         if (opResult.isPresent()) {
 895             return opResult.get();
 896         }
 897         return null;
 898     }
 899
 900     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 901         if (lri == null) {
 902             return true;
 903         }
 904
 905         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 906         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 907             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 908
 909         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 910             ? lri.getVpnInstanceList() : new ArrayList<>();
 911         if (vpnInstancesList.contains(vpnInstanceName)) {
 912             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 913             vpnInstancesList.remove(vpnInstanceName);
 914         }
 915         if (vpnInstancesList.isEmpty()) {
 916             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 917             if (tx != null) {
 918                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 919             } else {
 920                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 921             }
 922             return true;
 923         } else {
 924             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 925             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 926             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
 927         }
 928         return false;
 929     }
 930
 931     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 932                                       WriteTransaction tx) {
 933         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 934
 935         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 936
 937         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 938             dpId, label, groupId);
 939     }
 940
 941     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 942                                                 WriteTransaction tx) {
 943         List<MatchInfo> mkMatches = new ArrayList<>();
 944
 945         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 946             destDpId, label, actionsInfos);
 947
 948         // Matching metadata
 949         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 950         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 951
 952         List<InstructionInfo> mkInstructions = new ArrayList<>();
 953         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 954
 955         FlowEntity terminatingServiceTableFlowEntity =
 956             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 957             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
 958                 String.format("%s:%d", "TST Flow Entry ", label),
 959             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 960
 961         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 962
 963         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 964
 965         Node nodeDpn = FibUtil.buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 966         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 967             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 968             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
 969             .child(Flow.class, flowKey).build();
 970         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),
 971                 WriteTransaction.CREATE_MISSING_PARENTS);
 972     }
 973
 974     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 975         FlowEntity flowEntity;
 976         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
 977         List<MatchInfo> mkMatches = new ArrayList<>();
 978         // Matching metadata
 979         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 980         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 981             NwConstants.INTERNAL_TUNNEL_TABLE,
 982             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
 983             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
 984             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 985         Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId());
 986         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 987         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 988             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 989             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 990
 991         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 992         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 993     }
 994
 995     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 996         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 997         Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 998         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 999         boolean isExtraroute = false;
1000         if (localNextHopInfo == null) {
1001             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1002             if (usedRds.size() > 1) {
1003                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1004                         vrfEntry.getDestPrefix(), vpnName, rd);
1005                 return returnLocalDpnId;
1006             }
1007             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1008             //in the vpn
1009             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1010                     vpnName, rd, vrfEntry.getDestPrefix());
1011             if (extraRouteOptional.isPresent()) {
1012                 isExtraroute = true;
1013                 Routes extraRoute = extraRouteOptional.get();
1014                 String ipPrefix;
1015                 if (isIpv4Address(extraRoute.getNexthopIpList().get(0))) {
1016                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
1017                 } else {
1018                     ipPrefix = extraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX;
1019                 }
1020                 localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1021                 if (localNextHopInfo != null) {
1022                     String localNextHopIP = localNextHopInfo.getIpAddress();
1023                     BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1024                             vpnId, rd, vrfEntry, isExtraroute, vpnId /*parentVpnId*/);
1025                     if (!dpnId.equals(BigInteger.ZERO)) {
1026                         LOG.trace("Deleting ECMP group for prefix {}, dpn {}", vrfEntry.getDestPrefix(), dpnId);
1027                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1028                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ Collections.emptyList(),
1029                                 /*remove*/ false);
1030                         returnLocalDpnId.add(dpnId);
1031                     }
1032                 } else {
1033                     LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
1034                             + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
1035                 }
1036             }
1037
1038             if (localNextHopInfo == null) {
1039                 /* Imported VRF entry */
1040                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1041                 if (optionalLabel.isPresent()) {
1042                     Long label = optionalLabel.get();
1043                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1044                     LabelRouteInfo lri = getLabelRouteInfo(label);
1045                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1046                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1047                         prefixBuilder.setDpnId(lri.getDpnId());
1048                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0),
1049                                 vpnId, rd, vrfEntry, isExtraroute, lri.getParentVpnid());
1050                         if (!dpnId.equals(BigInteger.ZERO)) {
1051                             returnLocalDpnId.add(dpnId);
1052                         }
1053                     }
1054                 }
1055             }
1056
1057         } else {
1058             LOG.debug("Obtained prefix to interface for rd {} prefix {}", rd, vrfEntry.getDestPrefix());
1059             String localNextHopIP = localNextHopInfo.getIpAddress();
1060             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1061                 vpnId, rd, vrfEntry, isExtraroute, vpnId /*parentVpnId*/);
1062             if (!dpnId.equals(BigInteger.ZERO)) {
1063                 returnLocalDpnId.add(dpnId);
1064             }
1065         }
1066
1067         return returnLocalDpnId;
1068     }
1069
1070     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1071                                                 final Long vpnId, final String rd, final VrfEntry vrfEntry,
1072                                                 boolean isExtraroute, final Long parentVpnId) {
1073         if (localNextHopInfo != null) {
1074             final BigInteger dpnId = localNextHopInfo.getDpnId();
1075             if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) {
1076                 LOG.debug("checkDeleteLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1077                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1078                 return dpnId;
1079             }
1080             if (Prefixes.PrefixCue.PhysNetFunc.equals(localNextHopInfo.getPrefixCue())) {
1081                 LOG.debug("checkDeleteLocalFibEntry: PNF Prefix {} with vpnId {} rd {}. Skip local dpn {}"
1082                         + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
1083                 return dpnId;
1084             }
1085
1086             jobCoordinator.enqueueJob(FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId,
1087                     vrfEntry.getDestPrefix()), () -> {
1088                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1089                     baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1090                             NwConstants.DEL_FLOW, tx, null);
1091                     if (!fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(localNextHopInfo.getSubnetId(),
1092                             vpnId, rd)) {
1093                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1094                             FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1095                                 makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
1096                                         NwConstants.DEL_FLOW, tx);
1097                                 removeTunnelTableEntry(dpnId, label, tx);
1098                             });
1099                         }
1100                     }
1101                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1102                     futures.add(tx.submit());
1103                     return futures;
1104                 });
1105             //TODO: verify below adjacency call need to be optimized (?)
1106             //In case of the removal of the extra route, the loadbalancing group is updated
1107             if (!isExtraroute) {
1108                 baseVrfEntryHandler.deleteLocalAdjacency(dpnId, parentVpnId, localNextHopIP, vrfEntry.getDestPrefix());
1109             }
1110             return dpnId;
1111         }
1112         return BigInteger.ZERO;
1113     }
1114
1115     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, String rd,
1116             final VrfEntry vrfEntry, WriteTransaction tx) {
1117         Boolean wrTxPresent = true;
1118         if (tx == null) {
1119             wrTxPresent = false;
1120             tx = dataBroker.newWriteOnlyTransaction();
1121         }
1122
1123         String vpnName = fibUtil.getVpnNameFromId(vpnId);
1124         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1125                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1126
1127         List<AdjacencyResult> adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1128         if (adjacencyResults.isEmpty()) {
1129             LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}",
1130                     vrfEntry.getRoutePaths(), rd, remoteDpnId);
1131             LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
1132             return;
1133         }
1134
1135         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1136         List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1137                 vpnName, usedRds, vrfEntry.getDestPrefix());
1138         // create loadbalancing groups for extra routes only when the extra route is present behind
1139         // multiple VMs
1140         if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1
1141                 || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) {
1142             List<InstructionInfo> instructions = new ArrayList<>();
1143             // Obtain the local routes for this particular dpn.
1144             java.util.Optional<Routes> routes = vpnExtraRoutes
1145                     .stream()
1146                     .filter(route -> {
1147                         Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId,
1148                                 fibUtil.getIpPrefix(route.getNexthopIpList().get(0)));
1149                         if (prefixToInterface == null) {
1150                             return false;
1151                         }
1152                         return remoteDpnId.equals(prefixToInterface.getDpnId());
1153                     }).findFirst();
1154             long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1155                     routes.isPresent() ? routes.get() : null, vpnExtraRoutes);
1156             if (groupId == FibConstants.INVALID_GROUP_ID) {
1157                 LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1158                         vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1159                 return;
1160             }
1161             List<ActionInfo> actionInfos =
1162                     Collections.singletonList(new ActionGroup(groupId));
1163             instructions.add(new InstructionApplyActions(actionInfos));
1164             baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions,
1165                     NwConstants.ADD_FLOW, tx, null);
1166         } else {
1167             baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, null);
1168         }
1169
1170         if (!wrTxPresent) {
1171             tx.submit();
1172         }
1173         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1174     }
1175
1176     protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
1177     /* Get interface info from prefix to interface mapping;
1178         Use the interface info to get the corresponding vpn interface op DS entry,
1179         remove the adjacency corresponding to this fib entry.
1180         If adjacency removed is the last adjacency, clean up the following:
1181          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1182          - prefix to interface entry
1183          - vpn interface op DS
1184      */
1185         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1186         Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1187         if (prefixInfo == null) {
1188             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1189             String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
1190             Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
1191             if (extraRoute != null) {
1192                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1193                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1194                     if (nextHopIp != null) {
1195                         String ipPrefix;
1196                         if (isIpv4Address(nextHopIp)) {
1197                             ipPrefix = nextHopIp + NwConstants.IPV4PREFIX;
1198                         } else {
1199                             ipPrefix = nextHopIp + NwConstants.IPV6PREFIX;
1200                         }
1201                         prefixInfo = fibUtil.getPrefixToInterface(vpnId, ipPrefix);
1202                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1203                     }
1204                 }
1205             }
1206             if (prefixInfo == null) {
1207                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1208                 if (optionalLabel.isPresent()) {
1209                     Long label = optionalLabel.get();
1210                     List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1211                     LabelRouteInfo lri = getLabelRouteInfo(label);
1212                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1213                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1214                         prefixBuilder.setDpnId(lri.getDpnId());
1215                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1216                         prefixBuilder.setIpAddress(lri.getPrefix());
1217                         prefixInfo = prefixBuilder.build();
1218                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1219                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1220                         checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
1221                     }
1222                 }
1223             }
1224         } else {
1225             checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, null /*Routes*/);
1226         }
1227     }
1228
1229     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1230                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1231
1232         if (prefixInfo == null) {
1233             LOG.error("Cleanup VPN Data Failed as unable to find prefix Info for prefix {} VpnId {} rd {}",
1234                     vrfEntry.getDestPrefix(), vpnId, rd);
1235             return; //Don't have any info for this prefix (shouldn't happen); need to return
1236         }
1237
1238         if (Prefixes.PrefixCue.Nat.equals(prefixInfo.getPrefixCue())) {
1239             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1240                     vrfEntry.getDestPrefix(), vpnId, rd);
1241             return;
1242         }
1243
1244         String ifName = prefixInfo.getVpnInterfaceName();
1245         jobCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1246             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1247     }
1248
1249     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1250         Prefixes prefixInfo;
1251         Long vpnId;
1252         String rd;
1253         VrfEntry vrfEntry;
1254         Routes extraRoute;
1255
1256         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1257                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1258             this.prefixInfo = prefixInfo;
1259             this.vpnId = vpnId;
1260             this.rd = rd;
1261             this.vrfEntry = vrfEntry;
1262             this.extraRoute = extraRoute;
1263         }
1264
1265         @Override
1266         public List<ListenableFuture<Void>> call() throws Exception {
1267             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1268             // to call the respective helpers.
1269             return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1270                 //First Cleanup LabelRouteInfo
1271                 //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1272                 LOG.debug("cleanupVpnInterfaceWorker: rd {} prefix {}", rd, prefixInfo.getIpAddress());
1273                 if (VrfEntry.EncapType.Mplsgre.equals(vrfEntry.getEncapType())) {
1274                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1275                         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1276                         synchronized (label.toString().intern()) {
1277                             LabelRouteInfo lri = getLabelRouteInfo(label);
1278                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1279                                     && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1280                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1281                                         fibUtil.getVpnInstanceOpData(rd);
1282                                 String vpnInstanceName = "";
1283                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1284                                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1285                                 }
1286                                 boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, tx);
1287                                 if (lriRemoved) {
1288                                     String parentRd = lri.getParentVpnRd();
1289                                     fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1290                                             parentRd, vrfEntry.getDestPrefix()));
1291                                 }
1292                             } else {
1293                                 fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1294                                         rd, vrfEntry.getDestPrefix()));
1295                             }
1296                         }
1297                     });
1298                 }
1299                 String ifName = prefixInfo.getVpnInterfaceName();
1300                 Optional<String> optVpnName = fibUtil.getVpnNameFromRd(rd);
1301                 String vpnName = null;
1302
1303                 if (Prefixes.PrefixCue.PhysNetFunc.equals(prefixInfo.getPrefixCue())) {
1304                     /*Get vpnId for rd = networkId since op vpnInterface will be pointing to rd = networkId
1305                     * */
1306                     Optional<String> vpnNameOpt = fibUtil.getVpnNameFromRd(vrfEntry.getParentVpnRd());
1307                     if (vpnNameOpt.isPresent()) {
1308                         vpnId = fibUtil.getVpnId(vpnNameOpt.get());
1309                     }
1310                 }
1311                 if (optVpnName.isPresent()) {
1312                     vpnName = optVpnName.get();
1313                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = MDSALUtil
1314                             .read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1315                                     fibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1316                     if (opVpnInterface.isPresent()) {
1317                         long associatedVpnId = fibUtil.getVpnId(vpnName);
1318                         if (vpnId != associatedVpnId) {
1319                             LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}",
1320                                     vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1321                             LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1322                             return;
1323                         } else {
1324                             LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1325                                     vrfEntry.getDestPrefix(), associatedVpnId);
1326                         }
1327                     }
1328                 }
1329                 if (extraRoute != null) {
1330                     List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1331                     //Only one used Rd present in case of removal event
1332                     String usedRd = usedRds.get(0);
1333                     if (optVpnName.isPresent()) {
1334                         tx.delete(LogicalDatastoreType.OPERATIONAL,
1335                                 baseVrfEntryHandler.getVpnToExtrarouteIdentifier(vpnName, usedRd,
1336                                         vrfEntry.getDestPrefix()));
1337                         tx.delete(LogicalDatastoreType.CONFIGURATION,
1338                                 VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
1339                     }
1340                 }
1341                 Optional<AdjacenciesOp> optAdjacencies =
1342                     MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1343                                    FibUtil.getAdjListPathOp(ifName, vpnName));
1344                 int numAdj = 0;
1345                 if (optAdjacencies.isPresent()) {
1346                     numAdj = optAdjacencies.get().getAdjacency().size();
1347                 }
1348                 //remove adjacency corr to prefix
1349                 if (numAdj > 1) {
1350                     LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {} vpnName {}", vpnId,
1351                              vrfEntry.getDestPrefix(), vpnName);
1352                     tx.delete(LogicalDatastoreType.OPERATIONAL,
1353                               FibUtil.getAdjacencyIdentifierOp(ifName, vpnName, vrfEntry.getDestPrefix()));
1354                 } else {
1355                     //this is last adjacency (or) no more adjacency left for this vpn interface, so
1356                     //clean up the vpn interface from DpnToVpn list
1357                     LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.",
1358                              ifName, prefixInfo.getDpnId(), rd);
1359                     tx.delete(LogicalDatastoreType.OPERATIONAL,
1360                               FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName));
1361                 }
1362             }));
1363         }
1364     }
1365
1366     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1367         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1368         final String rd = vrfTableKey.getRouteDistinguisher();
1369         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
1370         if (vpnInstance == null) {
1371             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1372             return;
1373         }
1374         final Collection<VpnToDpnList> vpnToDpnList;
1375         if (vrfEntry.getParentVpnRd() != null
1376                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
1377             // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
1378             VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
1379             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
1380                     vpnInstance.getVpnToDpnList();
1381             LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}",
1382                     vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
1383         } else {
1384             vpnToDpnList = vpnInstance.getVpnToDpnList();
1385         }
1386
1387         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1388         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1389         List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1390         String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1391         if (subnetRoute != null) {
1392             long elanTag = subnetRoute.getElantag();
1393             LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}"
1394                     + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
1395             if (vpnToDpnList != null) {
1396                 jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()),
1397                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1398                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1399
1400                             baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1401                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
1402                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1403                                 optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null,
1404                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
1405                             }
1406
1407                             installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(),
1408                                     vrfEntry, NwConstants.DEL_FLOW, tx);
1409                         }
1410                     })));
1411             }
1412             optionalLabel.ifPresent(label -> {
1413                 synchronized (label.toString().intern()) {
1414                     LabelRouteInfo lri = getLabelRouteInfo(label);
1415                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1416                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1417                                 fibUtil.getVpnInstanceOpData(rd);
1418                         String vpnInstanceName = "";
1419                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1420                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1421                         }
1422                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1423                         if (lriRemoved) {
1424                             String parentRd = lri.getParentVpnRd();
1425                             fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1426                                     parentRd, vrfEntry.getDestPrefix()));
1427                             LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}"
1428                                     + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
1429                         }
1430                     } else {
1431                         fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(
1432                                 rd, vrfEntry.getDestPrefix()));
1433                         LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1434                                 label, rd, vrfEntry.getDestPrefix());
1435                     }
1436                 }
1437             });
1438             return;
1439         }
1440
1441         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1442             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1443         if (vpnToDpnList != null) {
1444             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1445                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1446             String jobKey;
1447             Optional<Routes> extraRouteOptional;
1448             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1449             if (usedRds != null && !usedRds.isEmpty()) {
1450                 jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
1451                 if (usedRds.size() > 1) {
1452                     LOG.error("The extra route prefix is still present in some DPNs");
1453                     return ;
1454                 } else {
1455                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1456                     //is not present in any other DPN
1457                     extraRouteOptional = VpnExtraRouteHelper
1458                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1459                 }
1460             } else {
1461                 jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
1462                 extraRouteOptional = Optional.absent();
1463             }
1464
1465             jobCoordinator.enqueueJob(jobKey,
1466                 () -> {
1467                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1468
1469                     if (localDpnIdList.size() <= 0) {
1470                         for (VpnToDpnList curDpn : vpnToDpnList) {
1471                             baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1472                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1473                         }
1474                     } else {
1475                         for (BigInteger localDpnId : localDpnIdList) {
1476                             for (VpnToDpnList curDpn : vpnToDpnList) {
1477                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1478                                     baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1479                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1480                                 }
1481                             }
1482                         }
1483                     }
1484                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1485                     futures.add(tx.submit());
1486                     return futures;
1487                 }, DJC_MAX_RETRIES);
1488         }
1489
1490         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1491         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1492         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1493
1494         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1495         // of the interVpnLink.
1496         Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
1497         if (optVpnUuid.isPresent()) {
1498             String vpnUuid = optVpnUuid.get();
1499             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1500                 Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1501                 if (optInterVpnLink.isPresent()) {
1502                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1503                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1504                         // This is route that points to the other endpoint of an InterVpnLink
1505                         // In that case, we should look for the FIB table pointing to
1506                         // LPortDispatcher table and remove it.
1507                         removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
1508                     }
1509                 }
1510             });
1511         }
1512
1513     }
1514
1515     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1516                                     int addOrRemove, WriteTransaction tx) {
1517         Boolean wrTxPresent = true;
1518         if (tx == null) {
1519             wrTxPresent = false;
1520             tx = dataBroker.newWriteOnlyTransaction();
1521         }
1522
1523         List<MatchInfo> matches = new ArrayList<>();
1524         matches.add(MatchEthernetType.MPLS_UNICAST);
1525         matches.add(new MatchMplsLabel(label));
1526
1527         // Install the flow entry in L3_LFIB_TABLE
1528         String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1529
1530         FlowEntity flowEntity;
1531         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1532             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1533         Flow flow = flowEntity.getFlowBuilder().build();
1534         String flowId = flowEntity.getFlowId();
1535         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1536         Node nodeDpn = FibUtil.buildDpnNode(dpId);
1537         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1538             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1539             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1540
1541         if (addOrRemove == NwConstants.ADD_FLOW) {
1542             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, WriteTransaction.CREATE_MISSING_PARENTS);
1543         } else {
1544             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1545         }
1546         if (!wrTxPresent) {
1547             tx.submit();
1548         }
1549
1550         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1551             dpId, label, instructions, flowKey, NwConstants.ADD_FLOW == addOrRemove ? "ADDED" : "REMOVED");
1552     }
1553
1554     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1555                                     final FutureCallback<List<Void>> callback) {
1556         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1557         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1558         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1559         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1560         List<SubTransaction> txnObjects =  new ArrayList<>();
1561         if (!vrfTable.isPresent()) {
1562             LOG.info("populateFibOnNewDpn: dpn: {}: VRF Table not yet available for RD {}", dpnId, rd);
1563             if (callback != null) {
1564                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1565                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1566                 Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1567             }
1568             return;
1569         }
1570
1571         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1572             () -> {
1573                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1574                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1575                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1576                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1577                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1578                         if (subnetRoute != null) {
1579                             long elanTag = subnetRoute.getElantag();
1580                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1581                             installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW, tx);
1582                             continue;
1583                         }
1584                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1585                         if (routerInt != null) {
1586                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1587                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1588                             routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1589                                     routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1590                                     NwConstants.ADD_FLOW);
1591                             continue;
1592                         }
1593                         //Handle local flow creation for imports
1594                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1595                             java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1596                             if (optionalLabel.isPresent()) {
1597                                 List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1598                                 LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1599                                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1600                                     if (lri.getDpnId().equals(dpnId)) {
1601                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1602                                         continue;
1603                                     }
1604                                 }
1605                             }
1606                         }
1607
1608                         boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1609                                 vrfEntry, dpnId);
1610                         if (shouldCreateRemoteFibEntry) {
1611                             LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}",
1612                                     vrfEntry, dpnId);
1613                             if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1614                                 bgpRouteVrfEntryHandler.createRemoteFibEntry(dpnId, vpnId,
1615                                         vrfTable.get().getRouteDistinguisher(), vrfEntry, tx, txnObjects);
1616                             } else {
1617                                 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
1618                                         vrfEntry, tx);
1619                             }
1620                         }
1621                     }
1622                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1623                     futures.add(tx.submit());
1624                 }
1625                 if (callback != null) {
1626                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1627                     Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1628                 }
1629                 return futures;
1630             });
1631     }
1632
1633     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1634                                             final String localNextHopIp, final String remoteNextHopIp) {
1635         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1636             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1637         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1638         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1639         List<SubTransaction> txnObjects =  new ArrayList<>();
1640         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1641         if (vrfTable.isPresent()) {
1642             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1643                 () -> {
1644                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1645                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1646                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1647                         vrfTable.get().getVrfEntry().stream()
1648                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1649                             .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId,
1650                                         rd, remoteNextHopIp, vrfTable, writeCfgTxn, txnObjects));
1651                         futures.add(writeCfgTxn.submit());
1652                     }
1653                     return futures;
1654                 });
1655         }
1656     }
1657
1658     public void manageRemoteRouteOnDPN(final boolean action,
1659                                        final BigInteger localDpnId,
1660                                        final long vpnId,
1661                                        final String rd,
1662                                        final String destPrefix,
1663                                        final String destTepIp,
1664                                        final long label) {
1665         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1666
1667         if (vpnInstance == null) {
1668             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1669             return;
1670         }
1671
1672         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, localDpnId),
1673             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1674                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1675                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1676                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1677                     if (vrfEntry == null) {
1678                         return;
1679                     }
1680                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1681                             action, localDpnId, vpnId, rd, destPrefix);
1682                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
1683                     VrfEntry modVrfEntry;
1684                     if (routePathList == null || routePathList.isEmpty()) {
1685                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
1686                                 Collections.singletonList(destTepIp),
1687                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
1688                     } else {
1689                         modVrfEntry = vrfEntry;
1690                     }
1691
1692                     if (action) {
1693                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1694                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey.getRouteDistinguisher(),
1695                                 modVrfEntry, tx);
1696                     } else {
1697                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1698                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1699                                 vrfEntry.getDestPrefix());
1700                         if (usedRds.size() > 1) {
1701                             LOG.debug("The extra route prefix is still present in some DPNs");
1702                             return;
1703                         }
1704                         //Is this fib route an extra route? If yes, get the nexthop which would be
1705                         //an adjacency in the vpn
1706                         Optional<Routes> extraRouteOptional = Optional.absent();
1707                         if (usedRds.size() != 0) {
1708                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1709                                     fibUtil.getVpnNameFromId(vpnInstance.getVpnId()),
1710                                     usedRds.get(0), vrfEntry.getDestPrefix());
1711                         }
1712                         baseVrfEntryHandler.deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
1713                                 extraRouteOptional, tx);
1714                     }
1715                 }
1716             })));
1717     }
1718
1719     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1720                                  final FutureCallback<List<Void>> callback) {
1721         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1722         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1723         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1724         List<SubTransaction> txnObjects =  new ArrayList<>();
1725         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1726         jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1727             () -> {
1728                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1729                 if (vrfTable.isPresent()) {
1730                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1731                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1732                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1733                                 /* Handle subnet routes here */
1734                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1735                             if (subnetRoute != null) {
1736                                 LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}"
1737                                         + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
1738                                 baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null,
1739                                         NwConstants.DEL_FLOW, tx, null);
1740                                 List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
1741                                 if (routePaths != null) {
1742                                     for (RoutePaths routePath : routePaths) {
1743                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
1744                                                 DEFAULT_FIB_FLOW_PRIORITY,
1745                                                 NwConstants.DEL_FLOW, tx);
1746                                         LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {} for"
1747                                                 + " rd {} prefix {}", routePath.getLabel(), rd,
1748                                                 vrfEntry.getDestPrefix());
1749                                     }
1750                                 }
1751                                 installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry,
1752                                         NwConstants.DEL_FLOW, tx);
1753                                 continue;
1754                             }
1755                             // ping responder for router interfaces
1756                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1757                             if (routerInt != null) {
1758                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1759                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1760                                 routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId,
1761                                         routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()),
1762                                         NwConstants.DEL_FLOW);
1763                                 continue;
1764                             }
1765
1766                             //Handle local flow deletion for imports
1767                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1768                                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1769                                 if (optionalLabel.isPresent()) {
1770                                     List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
1771                                     LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1772                                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1773                                         if (lri.getDpnId().equals(dpnId)) {
1774                                             deleteLocalFibEntry(vpnId, rd, vrfEntry);
1775                                             continue;
1776                                         }
1777                                     }
1778                                 }
1779                             }
1780
1781                             // Passing null as we don't know the dpn
1782                             // to which prefix is attached at this point
1783                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
1784                                     vrfEntry.getDestPrefix());
1785                             String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
1786                             Optional<Routes> extraRouteOptional;
1787                             //Is this fib route an extra route? If yes, get the nexthop which would be
1788                             //an adjacency in the vpn
1789                             if (usedRds != null && !usedRds.isEmpty()) {
1790                                 if (usedRds.size() > 1) {
1791                                     LOG.error("The extra route prefix is still present in some DPNs");
1792                                     return futures;
1793                                 } else {
1794                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1795                                             usedRds.get(0), vrfEntry.getDestPrefix());
1796
1797                                 }
1798                             } else {
1799                                 extraRouteOptional = Optional.absent();
1800                             }
1801                             if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1802                                 bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
1803                                         vrfEntry, extraRouteOptional, tx, txnObjects);
1804                             } else {
1805                                 baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(),
1806                                         vrfEntry, extraRouteOptional, tx);
1807                             }
1808                         }
1809                         futures.add(tx.submit());
1810                     }
1811                     if (callback != null) {
1812                         ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1813                         Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
1814                     }
1815                 }
1816                 return futures;
1817             });
1818
1819     }
1820
1821     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1822                                            final String localNextHopIp, final String remoteNextHopIp) {
1823         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
1824                 + " localNexthopIp {} , remoteNexhtHopIp {}",
1825             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1826         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1827         final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
1828         List<SubTransaction> txnObjects =  new ArrayList<>();
1829         final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1830         if (vrfTable.isPresent()) {
1831             jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId),
1832                 () -> {
1833                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1834                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1835                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1836                         vrfTable.get().getVrfEntry().stream()
1837                             .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
1838                             .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
1839                                 remoteNextHopIp, vrfTable, writeCfgTxn, txnObjects));
1840                         futures.add(writeCfgTxn.submit());
1841                     }
1842                     return futures;
1843                 });
1844         }
1845     }
1846
1847     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1848         InstanceIdentifierBuilder<VrfTables> idBuilder =
1849             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1850         return idBuilder.build();
1851     }
1852
1853     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
1854         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
1855                 .FLOWID_SEPARATOR + nextHop;
1856     }
1857
1858     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1859         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
1860                 + tableMiss + FLOWID_PREFIX;
1861     }
1862
1863     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
1864         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
1865             .child(VrfTables.class, new VrfTablesKey(rd))
1866             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
1867         Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1868         if (vrfEntry.isPresent()) {
1869             return vrfEntry.get();
1870         }
1871         return null;
1872     }
1873
1874     public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
1875                                              final String vpnName,
1876                                              final VrfEntry vrfEntry) {
1877         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
1878
1879         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
1880         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
1881
1882         if (targetDpns.isEmpty()) {
1883             LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
1884             return;
1885         }
1886
1887         java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
1888         java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1889
1890         // delete from FIB
1891         //
1892         optNextHop.ifPresent(nextHop -> {
1893             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
1894             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
1895             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
1896                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
1897
1898             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
1899             for (BigInteger dpId : targetDpns) {
1900                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
1901                           vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
1902
1903                 mdsalManager.removeFlow(dpId, flow);
1904             }
1905         });
1906
1907         // delete from LFIB
1908         //
1909         optLabel.ifPresent(label -> {
1910             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
1911
1912             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1913             for (BigInteger dpId : targetDpns) {
1914                 LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
1915                           vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
1916                 makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
1917             }
1918             tx.submit();
1919         });
1920     }
1921
1922     private boolean isPrefixAndNextHopPresentInLri(String prefix,
1923             List<String> nextHopAddressList, LabelRouteInfo lri) {
1924         return lri != null && lri.getPrefix().equals(prefix)
1925                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
1926     }
1927
1928     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
1929         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1930             return true;
1931         }
1932
1933         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1934         if (prefix != null) {
1935             BigInteger prefixDpnId = prefix.getDpnId();
1936             if (dpnId.equals(prefixDpnId)) {
1937                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
1938                         vrfEntry, dpnId);
1939                 return false;
1940             }
1941         }
1942         return true;
1943     }
1944 }