vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15 import java.math.BigInteger;
  16 import java.net.Inet4Address;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import java.util.concurrent.LinkedBlockingQueue;
  28 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  29 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  30 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  31 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  32 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  33 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  34 import org.opendaylight.genius.mdsalutil.ActionInfo;
  35 import org.opendaylight.genius.mdsalutil.ActionType;
  36 import org.opendaylight.genius.mdsalutil.FlowEntity;
  37 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  38 import org.opendaylight.genius.mdsalutil.InstructionType;
  39 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  40 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  41 import org.opendaylight.genius.mdsalutil.MatchInfo;
  42 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  45 import org.opendaylight.genius.mdsalutil.packet.IPProtocols;
  46 import org.opendaylight.genius.utils.ServiceIndex;
  47 import org.opendaylight.genius.utils.batching.ActionableResource;
  48 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  49 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  50 import org.opendaylight.genius.utils.batching.ResourceHandler;
  51 import org.opendaylight.genius.utils.batching.SubTransaction;
  52 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  53 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  54 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  55 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  56 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  57 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  58 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  59 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 119 import org.opendaylight.yangtools.yang.binding.DataObject;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.RpcResult;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> implements AutoCloseable, ResourceHandler {
 127     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 128     private static final String FLOWID_PREFIX = "L3.";
 129     private final DataBroker dataBroker;
 130     private final IMdsalApiManager mdsalManager;
 131     private IVpnManager vpnmanager;
 132     private final NexthopManager nextHopManager;
 133     private ItmRpcService itmManager;
 134     private final OdlInterfaceRpcService interfaceManager;
 135     private final IdManagerService idManager;
 136     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 137     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 138     private static final int LFIB_INTERVPN_PRIORITY = 1;
 139     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 140     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 141     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 142     List<SubTransaction> transactionObjects;
 143     private static final int PERIODICITY = 500;
 144     private static Integer batchSize;
 145     private static Integer batchInterval;
 146     private static final int BATCH_SIZE = 1000;
 147     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 148     private final ResourceBatchingManager resourceBatchingManager;
 149
 150     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 151                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 152                             final IdManagerService idManager) {
 153         super(VrfEntry.class, VrfEntryListener.class);
 154         this.dataBroker = dataBroker;
 155         this.mdsalManager = mdsalApiManager;
 156         this.nextHopManager = nexthopManager;
 157         this.interfaceManager = interfaceManager;
 158         this.idManager = idManager;
 159
 160         batchSize = Integer.getInteger("batch.size");
 161         if (batchSize == null) {
 162             batchSize = BATCH_SIZE;
 163         }
 164         batchInterval = Integer.getInteger("batch.wait.time");
 165         if (batchInterval == null) {
 166             batchInterval = PERIODICITY;
 167         }
 168         resourceBatchingManager = ResourceBatchingManager.getInstance();
 169         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 170         transactionObjects = new ArrayList<>();
 171     }
 172
 173     public void start() {
 174         LOG.info("{} start", getClass().getSimpleName());
 175         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 176     }
 177
 178     @Override
 179     protected VrfEntryListener getDataTreeChangeListener() { return VrfEntryListener.this; }
 180
 181     @Override
 182     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 183         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 184     }
 185
 186     @Override
 187     public DataBroker getResourceBroker() {
 188         return dataBroker;
 189     }
 190
 191     @Override
 192     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 193         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 194         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 195         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 196                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 197         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 198             createFibEntries(identifier, vrfEntry);
 199         } else {
 200             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 201             actResource.setAction(ActionableResource.CREATE);
 202             actResource.setInstanceIdentifier(identifier);
 203             actResource.setInstance(vrfEntry);
 204             vrfEntryBufferQ.add(actResource);
 205             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 206         }
 207         LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 208                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 209     }
 210
 211     @Override
 212     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 213         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 214         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 215         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 216                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 217         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 218             deleteFibEntries(identifier, vrfEntry);
 219         } else {
 220             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 221             actResource.setAction(ActionableResource.DELETE);
 222             actResource.setInstanceIdentifier(identifier);
 223             actResource.setInstance(vrfEntry);
 224             vrfEntryBufferQ.add(actResource);
 225             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 226         }
 227         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 228                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 229     }
 230
 231     @Override
 232     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 233         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 234
 235         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 236         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 237         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 238                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 239         // Handle BGP Routes first
 240         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 241             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 242             actResource.setAction(ActionableResource.UPDATE);
 243             actResource.setInstanceIdentifier(identifier);
 244             actResource.setInstance(update);
 245             actResource.setOldInstance(original);
 246             vrfEntryBufferQ.add(actResource);
 247             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 248                     rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 249             return;
 250         }
 251
 252         // Handle Internal Routes next (ie., STATIC only)
 253           if (FibUtil.isControllerManagedNonInterVpnLinkRoute(RouteOrigin.value(update.getOrigin()))) {
 254             SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 255             /* Ignore SubnetRoute entry, as it will be driven by createFibEntries call down below */
 256             if (subnetRoute == null) {
 257                 List<String> origNhList = original.getNextHopAddressList();
 258                 List<String> updateNhList = update.getNextHopAddressList();
 259                 //final SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
 260                 LOG.info("UPDATE: Original nexthop {} updateNextHop {} ", origNhList, updateNhList);
 261
 262                 // If original VRF Entry had nexthop null , but update VRF Entry
 263                 // has nexthop , route needs to be created on remote Dpns
 264                 if (((origNhList == null) || (origNhList.isEmpty()) &&
 265                         (updateNhList != null) && (!updateNhList.isEmpty()))) {
 266                     // TODO(vivek): Though ugly, Not handling this code now, as each
 267                     // tep add event will invoke flow addition
 268                     LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.", update.getDestPrefix());
 269                     return;
 270                 }
 271
 272                 // If original VRF Entry had valid nexthop , but update VRF Entry
 273                 // has nexthop empty'ed out, route needs to be removed from remote Dpns
 274                 if (((updateNhList == null) || (updateNhList.isEmpty()) &&
 275                         (origNhList != null) && (!origNhList.isEmpty()))) {
 276                     LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.", update.getDestPrefix());
 277                     return;
 278                 }
 279             }
 280             createFibEntries(identifier, update);
 281             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 282                     rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 283             return;
 284         }
 285
 286         /* Handl all other route origins */
 287         createFibEntries(identifier, update);
 288
 289         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 290                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 291     }
 292
 293     @Override
 294     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
 295         this.transactionObjects = transactionObjects;
 296         if (vrfEntry instanceof VrfEntry) {
 297             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 298         }
 299     }
 300
 301     @Override
 302     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
 303         this.transactionObjects = transactionObjects;
 304         if (vrfEntry instanceof VrfEntry) {
 305             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 306         }
 307     }
 308
 309     @Override
 310     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 311                        Object update, List<SubTransaction> transactionObjects) {
 312         this.transactionObjects = transactionObjects;
 313         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 314             createFibEntries(tx, identifier, (VrfEntry)update);
 315         }
 316     }
 317
 318     @Override
 319     public int getBatchSize() {
 320         return batchSize;
 321     }
 322
 323     @Override
 324     public int getBatchInterval() {
 325         return batchInterval;
 326     }
 327
 328     @Override
 329     public LogicalDatastoreType getDatastoreType() {
 330         return LogicalDatastoreType.CONFIGURATION;
 331     }
 332
 333     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 334         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 335
 336         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 337         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 338         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 339
 340         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 341         final Long vpnId = vpnInstance.getVpnId();
 342         final String rd = vrfTableKey.getRouteDistinguisher();
 343         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 344         if (subnetRoute != null) {
 345             final long elanTag = subnetRoute.getElantag();
 346             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 347                     rd, vrfEntry.getDestPrefix(), elanTag);
 348             if (vpnToDpnList != null) {
 349                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 350                 dataStoreCoordinator.enqueueJob("FIB-"+ rd.toString() + "-" + vrfEntry.getDestPrefix(),
 351                         new Callable<List<ListenableFuture<Void>>>() {
 352                             @Override
 353                             public List<ListenableFuture<Void>> call() throws Exception {
 354                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 355                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 356                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 357                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 358                                     }
 359                                 }
 360                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 361                                 futures.add(tx.submit());
 362                                 return futures;
 363                             }
 364                         });
 365             }
 366             return;
 367         }
 368         // ping responder for router interfaces
 369         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 370             return;
 371         }
 372
 373         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 374
 375         if (vpnToDpnList != null) {
 376             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 377             dataStoreCoordinator.enqueueJob("FIB-"+ rd.toString() + "-" + vrfEntry.getDestPrefix(),
 378                     new Callable<List<ListenableFuture<Void>>>() {
 379                         @Override
 380                         public List<ListenableFuture<Void>> call() throws Exception {
 381                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 382                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 383                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 384                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 385                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 386                                     }
 387                                 }
 388                             }
 389                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 390                             futures.add(tx.submit());
 391                             return futures;
 392                         }
 393                     });
 394         }
 395
 396         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 397         if ( optVpnUuid.isPresent() ) {
 398             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 399             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 400             if ( optInterVpnLink.isPresent() ) {
 401                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 402                 String vpnUuid = optVpnUuid.get();
 403                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 404                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid) ) {
 405                     // This is an static route that points to the other endpoint of an InterVpnLink
 406                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 407                     installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 408                     installInterVpnRouteInLFib(rd, vrfEntry);
 409                 }
 410             }
 411         }
 412     }
 413
 414
 415     /*
 416       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 417       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 418       provided by ResourceBatchingManager
 419      */
 420     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 421         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 422
 423         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 424         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 425         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 426
 427         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 428         final String rd = vrfTableKey.getRouteDistinguisher();
 429         if (vpnToDpnList != null) {
 430             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 431                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 432                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 433                 }
 434             }
 435         }
 436     }
 437
 438     // FIXME: Refactoring needed here.
 439     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 440     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 441                                    int addOrRemove) {
 442         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 443         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 444
 445         String rd = vrfTableKey.getRouteDistinguisher();
 446         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 447         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 448             if (vpnInstance == null) {
 449                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 450                 return;
 451             }
 452         } else {
 453             Preconditions.checkNotNull(vpnInstance,
 454                                        "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 455         }
 456         String vpnUuid = vpnInstance.getVpnInstanceName();
 457         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 458                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 459
 460         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 461         // there an interVpnLink for the involved vpn in order to make learn the new route to
 462         // the other part of the inter-vpn-link.
 463
 464         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 465         Optional<InterVpnLink> interVpnLink =
 466             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 467                                                   : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 468         if ( !interVpnLink.isPresent() ) {
 469             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 470             return;
 471         }
 472
 473         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 474         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 475         boolean proceed =
 476             (addOrRemove == NwConstants.DEL_FLOW) || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 477                                                        && interVpnLink.get().isBgpRoutesLeaking() );
 478
 479         if ( proceed ) {
 480             boolean isVpnFirstEndpoint = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid);
 481
 482             String theOtherVpnId = isVpnFirstEndpoint ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 483                                                       : vpnUuid;
 484             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 485             String endpointIp = isVpnFirstEndpoint ? interVpnLink.get().getFirstEndpoint().getIpAddress().getValue()
 486                                                    : interVpnLink.get().getSecondEndpoint().getIpAddress().getValue();
 487
 488             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 489                     InstanceIdentifier.builder(FibEntries.class)
 490                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 491                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 492                             .build();
 493             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 494                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 495                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 496                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 497                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 498                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 499                         .setLabel(label)
 500                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 501                         .build();
 502                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 503             } else {
 504                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 505                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 506             }
 507         }
 508     }
 509
 510     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 511         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 512         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 513         prefixBuilder.setDpnId(lri.getDpnId());
 514         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 515         prefixBuilder.setIpAddress(lri.getPrefix());
 516         // Increment the refCount here
 517         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 518                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 519         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 520         if (!isPresentInList) {
 521             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 522             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 523             vpnInstanceNames.add(vpnInstanceName);
 524             builder.setVpnInstanceList(vpnInstanceNames);
 525             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 526         } else {
 527             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 528         }
 529         return prefixBuilder.build();
 530     }
 531
 532     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 533                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 534         Boolean wrTxPresent = true;
 535         if (tx == null) {
 536             wrTxPresent = false;
 537             tx = dataBroker.newWriteOnlyTransaction();
 538         }
 539         synchronized (vrfEntry.getLabel().toString().intern()) {
 540             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 541             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 542                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 543
 544                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 545                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 546                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 547                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 548                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 549                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 550                         }
 551                     }
 552                 }
 553                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 554                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 555             }
 556         }
 557         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 558         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 559         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 560         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 561         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 562
 563         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 564             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 565             // reinitialize instructions list for LFIB Table
 566             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 567
 568             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 569             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 570             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 571             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 572
 573             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 574         }
 575         if (!wrTxPresent ) {
 576             tx.submit();
 577         }
 578     }
 579
 580     /*
 581      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 582      * LportDispatcher table (via table 80)
 583      */
 584     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 585         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 586         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 587         // packet is commuted from Vpn2 to Vpn1.
 588         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 589         if ( !vpnNameOpc.isPresent() ) {
 590             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 591             return;
 592         }
 593
 594         String vpnName = vpnNameOpc.get();
 595         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 596         boolean interVpnLinkFound = false;
 597         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 598             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 599             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 600                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 601             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 602                 interVpnLinkFound = true;
 603
 604                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 605                 if ( !vpnLinkState.isPresent()
 606                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 607                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 608                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 609                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 610                     return;
 611                 }
 612
 613                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 614                                                                : vpnLinkState.get().getSecondEndpointState().getDpId();
 615                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 616                                                  : vpnLinkState.get().getFirstEndpointState().getLportTag();
 617
 618                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 619
 620                 for ( BigInteger dpId : targetDpns ) {
 621                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 622
 623                     BigInteger[] metadata = new BigInteger[] {
 624                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 625                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 626                     };
 627                     List<InstructionInfo> instructions =
 628                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 629                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 630                                     new InstructionInfo(InstructionType.goto_table,
 631                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 632
 633                     LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 634                               vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 635                               dpId, interVpnLink.getName());
 636
 637                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 638                             NwConstants.ADD_FLOW, null);
 639                 }
 640
 641                 break;
 642             }
 643         }
 644
 645         if ( !interVpnLinkFound ) {
 646             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 647                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 648         }
 649     }
 650
 651
 652     /*
 653      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 654      */
 655     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 656                                                final VrfEntry vrfEntry, long vpnTag) {
 657         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 658         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 659                 && vrfEntry.getNextHopAddressList().size() == 1);
 660         String destination = vrfEntry.getDestPrefix();
 661         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 662         String iVpnLinkName = interVpnLink.getInterVpnLinkName();
 663
 664         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 665         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 666         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 667         if ( interVpnLink.getState().or(State.Error) != State.Active ) {
 668             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 669                     destination, nextHop, iVpnLinkName);
 670             return;
 671         }
 672
 673         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 674         if ( !optOtherEndpointLportTag.isPresent() ) {
 675             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 676                      vpnUuid, iVpnLinkName);
 677             return;
 678         }
 679
 680         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 681         if ( targetDpns.isEmpty() ) {
 682             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, iVpnLinkName);
 683             return;
 684         }
 685
 686         BigInteger[] metadata = new BigInteger[] {
 687                 MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 688                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 689                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 690         };
 691         List<Instruction> instructions =
 692                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 693                               new InstructionInfo(InstructionType.goto_table,
 694                                                   new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 695
 696         String values[] = destination.split("/");
 697         String destPrefixIpAddress = values[0];
 698         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 699
 700         List<MatchInfo> matches = new ArrayList<>();
 701         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 702                                                                               MetaDataUtil.METADATA_MASK_VRFID }));
 703         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 704
 705         if (prefixLength != 0) {
 706             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 707                                       new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 708         }
 709
 710         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 711         String flowRef = getInterVpnFibFlowRef(iVpnLinkName, destination, nextHop);
 712         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 713                                                  COOKIE_VM_FIB_TABLE, matches, instructions);
 714
 715         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}" ,
 716                   vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 717
 718         for ( BigInteger dpId : targetDpns ) {
 719
 720             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB",
 721                      vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 722                      dpId, interVpnLink.getInterVpnLinkName());
 723
 724             mdsalManager.installFlow(dpId, flowEntity);
 725         }
 726     }
 727
 728
 729     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 730                                                      InstanceIdentifier<T> path) {
 731
 732         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 733
 734         Optional<T> result = Optional.absent();
 735         try {
 736             result = tx.read(datastoreType, path).get();
 737         } catch (Exception e) {
 738             throw new RuntimeException(e);
 739         }
 740
 741         return result;
 742     }
 743
 744     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 745         List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 746         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 747
 748         if (localNextHopInfo == null) {
 749             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 750             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 751             if (extraRoute != null) {
 752                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 753                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 754                     if (nextHopIp != null) {
 755                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp + "/32");
 756                         if (localNextHopInfo != null) {
 757                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 758                         }
 759                     }
 760                 }
 761             }
 762         } else {
 763             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 764         }
 765
 766         return returnLocalDpnId;
 767     }
 768
 769     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 770         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 771         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 772         String localNextHopIP = vrfEntry.getDestPrefix();
 773
 774         if (localNextHopInfo == null) {
 775             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 776             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 777             if (extraRoute != null) {
 778                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 779                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 780                     if (nextHopIp != null) {
 781                         localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
 782                         localNextHopIP = nextHopIp + "/32";
 783                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 784                         returnLocalDpnId.add(dpnId);
 785                     }
 786                 }
 787             }
 788             if (localNextHopInfo == null) {
 789             /* imported routes case */
 790                 synchronized (vrfEntry.getLabel().toString().intern()) {
 791                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 792                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 793                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 794                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 795                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 796                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 797                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 798                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 799                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 800                                     localNextHopIP = lri.getPrefix();
 801                                 } else {
 802                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 803                                     localNextHopIP = lri.getPrefix();
 804                                 }
 805                             }
 806                             if (localNextHopInfo != null) {
 807                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 808                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 809                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 810                                 returnLocalDpnId.add(dpnId);
 811                             }
 812                         }
 813                     }
 814                 }
 815             }
 816         } else {
 817             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 818             returnLocalDpnId.add(dpnId);
 819         }
 820
 821         return returnLocalDpnId;
 822     }
 823
 824     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 825                                                 final VrfEntry vrfEntry, Long parentVpnId){
 826         if (localNextHopInfo != null) {
 827             final BigInteger dpnId = localNextHopInfo.getDpnId();
 828             if (!isVpnPresentInDpn(rd, dpnId)) {
 829                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 830                 return BigInteger.ZERO;
 831             }
 832
 833             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 834                     localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix());
 835             if (groupId == 0) {
 836                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 837                         vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 838                 return BigInteger.ZERO;
 839             }
 840             List<ActionInfo> actionsInfos =
 841                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 842             final List<InstructionInfo> instructions =
 843                     Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 844             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 845                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 846             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 847             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 848                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 849                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 850             } else {
 851                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 852             }
 853             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 854             dataStoreCoordinator.enqueueJob("FIB-"+ vpnId.toString() + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
 855                     new Callable<List<ListenableFuture<Void>>>() {
 856                         @Override
 857                         public List<ListenableFuture<Void>> call() throws Exception {
 858                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 859                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 860                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 861                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 862                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 863                             }
 864                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 865                             futures.add(tx.submit());
 866                             return futures;
 867                         }
 868                     });
 869             return dpnId;
 870         }
 871         return BigInteger.ZERO;
 872     }
 873
 874     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 875         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 876         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 877         if (dpnInVpn.isPresent()) {
 878             return true;
 879         }
 880         return false;
 881     }
 882
 883     private LabelRouteInfo getLabelRouteInfo(Long label) {
 884         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 885                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 886         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 887         if (opResult.isPresent()) {
 888             return opResult.get();
 889         }
 890         return null;
 891     }
 892
 893     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 894         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 895         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 896                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 897         if (lri == null) {
 898             return true;
 899         }
 900         List<String> vpnInstancesList = lri.getVpnInstanceList() != null ? lri.getVpnInstanceList() : new ArrayList<String>();
 901         if (vpnInstancesList.contains(vpnInstanceName)) {
 902             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 903             vpnInstancesList.remove(vpnInstanceName);
 904         }
 905         if (vpnInstancesList.size() == 0) {
 906             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 907             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 908             return true;
 909         } else {
 910             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 911             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 912             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 913         }
 914         return false;
 915     }
 916
 917     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 918                                       WriteTransaction tx) {
 919         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 920         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 921
 922
 923         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 924
 925         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 926                 dpId, label, groupId);
 927     }
 928
 929     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 930                                                  WriteTransaction tx) {
 931         List<MatchInfo> mkMatches = new ArrayList<>();
 932
 933         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 934
 935         // Matching metadata
 936         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 937         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 938
 939         List<InstructionInfo> mkInstructions = new ArrayList<>();
 940         mkInstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 941
 942         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 943                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 944                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 945
 946         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 947
 948         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 949
 950         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 951         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 952                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 953                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 954         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 955     }
 956
 957     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 958         FlowEntity flowEntity;
 959         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 960         List<MatchInfo> mkMatches = new ArrayList<>();
 961         // Matching metadata
 962         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 963         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 964                 NwConstants.INTERNAL_TUNNEL_TABLE,
 965                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 966                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 967                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 968         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 969         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 970         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 971                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 972                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 973
 974         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 975         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 976     }
 977
 978     /**
 979      * Delete local FIB entry
 980      * @param vpnId
 981      * @param rd
 982      * @param vrfEntry
 983      * @return
 984      */
 985     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 986         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 987         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 988         String localNextHopIP = vrfEntry.getDestPrefix();
 989
 990         if (localNextHopInfo == null) {
 991             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 992             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 993             if (extra_route != null) {
 994                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 995                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 996                     if (nextHopIp != null) {
 997                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 998                         localNextHopIP = nextHopIp + "/32";
 999                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1000                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
1001                         if (!dpnId.equals(BigInteger.ZERO)) {
1002                             returnLocalDpnId.add(dpnId);
1003                         }
1004                     }
1005                 }
1006             }
1007
1008             if (localNextHopInfo == null) {
1009               /* Imported VRF entry */
1010                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1011                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1012                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1013                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
1014                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
1015                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
1016                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1017                     if (!dpnId.equals(BigInteger.ZERO)) {
1018                         returnLocalDpnId.add(dpnId);
1019                     }
1020                 }
1021             }
1022
1023
1024         } else {
1025             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1026                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1027             if (!dpnId.equals(BigInteger.ZERO)) {
1028                 returnLocalDpnId.add(dpnId);
1029             }
1030         }
1031
1032         return returnLocalDpnId;
1033     }
1034
1035     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1036                                                 final Long vpnId, final String rd,
1037                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1038         if (localNextHopInfo != null) {
1039             final BigInteger dpnId = localNextHopInfo.getDpnId();;
1040             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1041             dataStoreCoordinator.enqueueJob("FIB-"+ vpnId.toString() + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1042                     new Callable<List<ListenableFuture<Void>>>() {
1043                         @Override
1044                         public List<ListenableFuture<Void>> call() throws Exception {
1045                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1046                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1047                                     NwConstants.DEL_FLOW, tx);
1048                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1049                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1050                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1051                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1052                             }
1053                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1054                             futures.add(tx.submit());
1055                             return futures;
1056                         }
1057                     });
1058             //TODO: verify below adjacency call need to be optimized (?)
1059             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1060             return dpnId;
1061         }
1062         return BigInteger.ZERO;
1063     }
1064
1065     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1066         return InstanceIdentifier.builder(VpnToExtraroute.class)
1067                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1068                         new ExtrarouteKey(ipPrefix)).build();
1069     }
1070
1071     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1072         Optional<Extraroute> extraRouteInfo =
1073                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1074         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1075
1076     }
1077
1078     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1079         try {
1080             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1081                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1082             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1083             if(!rpcResult.isSuccessful()) {
1084                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1085             } else {
1086                 return rpcResult.getResult().getTunnelType();
1087             }
1088
1089         } catch (InterruptedException | ExecutionException e) {
1090             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1091         }
1092
1093         return null;
1094
1095     }
1096     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1097                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1098         Boolean wrTxPresent = true;
1099         if (tx == null) {
1100             wrTxPresent = false;
1101             tx = dataBroker.newWriteOnlyTransaction();
1102         }
1103         String rd = vrfTableKey.getRouteDistinguisher();
1104         LOG.debug(  "createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1105                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1106         /********************************************/
1107         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1108
1109         if (adjacencyResults.isEmpty()) {
1110             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1111                     vrfEntry.getNextHopAddressList(), rd);
1112             LOG.warn("Failed to add Route: {} in vpn: {}",
1113                     vrfEntry.getDestPrefix(), rd);
1114             return;
1115         }
1116
1117         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1118             List<InstructionInfo> instructions = new ArrayList<>();
1119             List<ActionInfo> actionInfos = new ArrayList<>();
1120             String egressInterface = adjacencyResult.getInterfaceName();
1121             if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
1122                 addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
1123             } else {
1124                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1125             }
1126             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface);
1127             if (egressActions.isEmpty()) {
1128                 LOG.error(
1129                         "Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.",
1130                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface);
1131                 return;
1132             }
1133             actionInfos.addAll(egressActions);
1134             instructions.add(new InstructionInfo(InstructionType.apply_actions, actionInfos));
1135             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1136         }
1137         if(!wrTxPresent ){
1138             tx.submit();
1139         }
1140         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1141     }
1142
1143     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1144         String ipPrefix = vrfEntry.getDestPrefix();
1145         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1146         if (prefixInfo == null) {
1147             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1148             return;
1149         }
1150
1151         String ifName = prefixInfo.getVpnInterfaceName();
1152         if (ifName == null) {
1153             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1154             return;
1155         }
1156
1157         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1158         if (macAddress == null) {
1159             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1160             return;
1161         }
1162
1163         actionInfos.add(new ActionInfo(ActionType.set_field_eth_dest, new String[] { macAddress }, actionInfos.size()));
1164     }
1165
1166     private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
1167             List<ActionInfo> actionInfos) {
1168         Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1169         if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1170             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1171             actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[] { null }));
1172             actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label,
1173                     new String[] { Long.toString(vrfEntry.getLabel()) }));
1174             actionInfos.add(new ActionInfo(ActionType.nx_load_in_port, new BigInteger[]{BigInteger.ZERO}));
1175         } else {
1176             int label = vrfEntry.getLabel().intValue();
1177             BigInteger tunnelId;
1178             // FIXME vxlan vni bit set is not working properly with OVS.need to
1179             // revisit
1180             if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1181                 tunnelId = BigInteger.valueOf(label);
1182             } else {
1183                 tunnelId = BigInteger.valueOf(label);
1184             }
1185
1186             LOG.debug("adding set tunnel id action for label {}", label);
1187             actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[] { tunnelId }));
1188             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1189         }
1190     }
1191
1192     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1193         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1194         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1195         if (dpnInVpn.isPresent()) {
1196             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1197             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1198
1199             if (vpnInterfaces.remove(currVpnInterface)) {
1200                 if (vpnInterfaces.isEmpty()) {
1201                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1202                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1203                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1204                 } else {
1205                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1206                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1207                             VpnInterfaces.class,
1208                             new VpnInterfacesKey(intfName)));
1209                 }
1210             }
1211         }
1212     }
1213
1214     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1215     /* Get interface info from prefix to interface mapping;
1216         Use the interface info to get the corresponding vpn interface op DS entry,
1217         remove the adjacency corresponding to this fib entry.
1218         If adjacency removed is the last adjacency, clean up the following:
1219          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1220          - prefix to interface entry
1221          - vpn interface op DS
1222      */
1223         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1224         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1225         Extraroute extraRoute = null;
1226         if (prefixInfo == null) {
1227             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1228             if(extraRoute != null) {
1229                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1230                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1231
1232                     if (nextHopIp != null) {
1233                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
1234                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1235                     }
1236                 }
1237             }
1238             if (prefixInfo == null) {
1239                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1240                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1241                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1242                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1243                     prefixBuilder.setDpnId(lri.getDpnId());
1244                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1245                     prefixBuilder.setIpAddress(lri.getPrefix());
1246                     prefixInfo = prefixBuilder.build();
1247                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1248                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1249                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1250                 }
1251             }
1252         } else {
1253             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1254         }
1255     }
1256
1257     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1258                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1259
1260         if (prefixInfo == null) {
1261             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1262             return; //Don't have any info for this prefix (shouldn't happen); need to return
1263         }
1264
1265         String ifName = prefixInfo.getVpnInterfaceName();
1266         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1267         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1268                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1269     }
1270
1271     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1272         Prefixes prefixInfo;
1273         Long vpnId;
1274         String rd;
1275         VrfEntry vrfEntry;
1276         Extraroute extraRoute;
1277
1278         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1279                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1280             this.prefixInfo = prefixInfo;
1281             this.vpnId = vpnId;
1282             this.rd= rd;
1283             this.vrfEntry= vrfEntry;
1284             this.extraRoute = extraRoute;
1285         }
1286
1287         @Override
1288         public List<ListenableFuture<Void>> call() throws Exception {
1289             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1290             // to call the respective helpers.
1291
1292             //First Cleanup LabelRouteInfo
1293             synchronized (vrfEntry.getLabel().toString().intern()) {
1294                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1295                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1296                                 vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1297                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1298                     String vpnInstanceName = "";
1299                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1300                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1301                         }
1302                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1303                     if (lriRemoved) {
1304                             String parentRd = lri.getParentVpnRd();
1305                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1306                                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1307                         }
1308                 } else {
1309                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1310                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1311                 }
1312             }
1313             String ifName = prefixInfo.getVpnInterfaceName();
1314             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1315                     FibUtil.getVpnInterfaceIdentifier(ifName));
1316             if (optvpnInterface.isPresent()) {
1317                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1318                 if (vpnId != associatedVpnId) {
1319                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1320                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1321                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1322                     return null;
1323                 } else {
1324                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1325                             vrfEntry.getDestPrefix(), associatedVpnId);
1326                 }
1327             }
1328             if (extraRoute != null) {
1329                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1330                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1331             }
1332             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1333                     FibUtil.getAdjListPath(ifName));
1334             int numAdj = 0;
1335             if (optAdjacencies.isPresent()) {
1336                 numAdj = optAdjacencies.get().getAdjacency().size();
1337             }
1338             //remove adjacency corr to prefix
1339             if (numAdj > 1) {
1340                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1341                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1342                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1343             }
1344             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1345                 //clean up the vpn interface from DpnToVpn list
1346                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1347                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1348                         FibUtil.getVpnInterfaceIdentifier(ifName));
1349             }
1350             return null;
1351         }
1352     }
1353
1354     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1355         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1356         final String rd  = vrfTableKey.getRouteDistinguisher();
1357         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1358         if (vpnInstance == null) {
1359             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1360             return;
1361         }
1362         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1363         long elanTag = 0L;
1364         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1365         if (subnetRoute != null) {
1366             elanTag = subnetRoute.getElantag();
1367             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1368                     rd, vrfEntry.getDestPrefix(), elanTag);
1369             if (vpnToDpnList != null) {
1370                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1371                 dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1372                         new Callable<List<ListenableFuture<Void>>>() {
1373                             @Override
1374                             public List<ListenableFuture<Void>> call() throws Exception {
1375                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1376
1377                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1378
1379                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1380                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1381                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1382                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1383                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1384                                     }
1385                                 }
1386                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1387                                 futures.add(tx.submit());
1388                                 return futures;
1389                             }
1390                         });
1391             }
1392             synchronized (vrfEntry.getLabel().toString().intern()) {
1393                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1394                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1395                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1396                     String vpnInstanceName = "";
1397                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1398                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1399                     }
1400                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1401                     if (lriRemoved) {
1402                         String parentRd = lri.getParentVpnRd();
1403                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1404                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1405                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1406                                 vrfEntry.getDestPrefix());
1407                     }
1408                 } else {
1409                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1410                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1411                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1412                             vrfEntry.getDestPrefix());
1413                 }
1414             }
1415             return;
1416         }
1417         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1418             return;
1419         }
1420
1421         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1422                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1423         if (vpnToDpnList != null) {
1424             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1425             dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
1426                     new Callable<List<ListenableFuture<Void>>>() {
1427                         @Override
1428                         public List<ListenableFuture<Void>> call() throws Exception {
1429                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1430
1431                             if (localDpnIdList.size() <= 0) {
1432                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1433                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1434                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1435                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1436                                         }
1437                                     } else {
1438                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1439                                     }
1440                                 }
1441                             } else {
1442                                 for (BigInteger localDpnId : localDpnIdList) {
1443                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1444                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1445                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1446                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1447                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1448                                                 }
1449                                             } else {
1450                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1451                                             }
1452                                         }
1453                                     }
1454                                 }
1455                             }
1456                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1457                             futures.add(tx.submit());
1458                             return futures;
1459                         }
1460                     });
1461         }
1462
1463         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1464         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1465         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1466
1467         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1468         // of the interVpnLink.
1469         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1470         if ( optVpnUuid.isPresent() ) {
1471             String vpnUuid = optVpnUuid.get();
1472             List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
1473             if(routeNexthoplist.isEmpty()) {
1474                 LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
1475                 return;
1476             }
1477             String routeNexthop = routeNexthoplist.get(0);
1478             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1479             if ( optInterVpnLink.isPresent() ) {
1480                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1481                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid))
1482                 {
1483                     // This is route that points to the other endpoint of an InterVpnLink
1484                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1485                     removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1486                                                  interVpnLink.isFirstEndpointVpnName(rd),
1487                                                  vrfEntry);
1488                 }
1489             }
1490         }
1491
1492     }
1493
1494     /*
1495       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1496       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1497       provided by ResourceBatchingManager
1498      */
1499     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1500         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1501
1502         final String rd  = vrfTableKey.getRouteDistinguisher();
1503         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1504         if (vpnInstance == null) {
1505             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1506             return;
1507         }
1508         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1509         if (vpnToDpnList != null) {
1510             for (VpnToDpnList curDpn : vpnToDpnList) {
1511                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1512                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1513                 }
1514             }
1515         }
1516     }
1517
1518     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1519                                   final long vpnId, final VrfTablesKey vrfTableKey,
1520                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1521
1522         Boolean wrTxPresent = true;
1523         if (tx == null) {
1524             wrTxPresent = false;
1525             tx = dataBroker.newWriteOnlyTransaction();
1526         }
1527
1528         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}", vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1529         String rd = vrfTableKey.getRouteDistinguisher();
1530
1531         if(localDpnId != null) {
1532             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1533             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1534             return;
1535         }
1536
1537         // below two reads are kept as is, until best way is found to identify dpnID
1538         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1539         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1540
1541         if (localNextHopInfo == null && extraRoute != null) {
1542             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1543             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1544                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1545                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1546             }
1547         } else {
1548             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1549         }
1550         if(!wrTxPresent ){
1551             tx.submit();
1552         }
1553     }
1554
1555     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1556                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1557         boolean isRemoteRoute = true;
1558         if (localNextHopInfo != null) {
1559             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1560         }
1561         if (isRemoteRoute) {
1562             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1563             return true;
1564         } else {
1565             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1566             return false;
1567         }
1568     }
1569
1570     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1571         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1572         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1573     }
1574
1575     private long get
1576             (byte[] rawIpAddress) {
1577         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1578                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1579     }
1580
1581     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1582                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1583         Boolean wrTxPresent = true;
1584         if (tx == null) {
1585             wrTxPresent = false;
1586             tx = dataBroker.newWriteOnlyTransaction();
1587         }
1588
1589         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1590         String values[] = vrfEntry.getDestPrefix().split("/");
1591         String ipAddress = values[0];
1592         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1593         if (addOrRemove == NwConstants.ADD_FLOW) {
1594             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1595         } else {
1596             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1597         }
1598         InetAddress destPrefix;
1599         try {
1600             destPrefix = InetAddress.getByName(ipAddress);
1601         } catch (UnknownHostException e) {
1602             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1603             return;
1604         }
1605
1606         List<MatchInfo> matches = new ArrayList<>();
1607
1608         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1609                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1610
1611         if (destPrefix instanceof Inet4Address) {
1612             matches.add(new MatchInfo(MatchFieldType.eth_type,
1613                     new long[] { NwConstants.ETHTYPE_IPV4 }));
1614             if(prefixLength != 0) {
1615                 matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1616                         destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1617             }
1618         } else {
1619             matches.add(new MatchInfo(MatchFieldType.eth_type,
1620                     new long[] { NwConstants.ETHTYPE_IPV6 }));
1621             if(prefixLength != 0) {
1622                 matches.add(new MatchInfo(MatchFieldType.ipv6_destination, new String[] {
1623                         destPrefix.getHostAddress() + "/" + Integer.toString(prefixLength)}));
1624             }
1625         }
1626
1627         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1628         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1629         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1630                 COOKIE_VM_FIB_TABLE, matches, instructions);
1631
1632         Flow flow = flowEntity.getFlowBuilder().build();
1633         String flowId = flowEntity.getFlowId();
1634         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1635         Node nodeDpn = buildDpnNode(dpId);
1636
1637         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1638                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1639                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1640
1641         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1642             SubTransaction subTransaction = new SubTransactionImpl();
1643             if (addOrRemove == NwConstants.ADD_FLOW) {
1644                 subTransaction.setInstanceIdentifier(flowInstanceId);
1645                 subTransaction.setInstance(flow);
1646                 subTransaction.setAction(SubTransaction.CREATE);
1647             } else {
1648                 subTransaction.setInstanceIdentifier(flowInstanceId);
1649                 subTransaction.setAction(SubTransaction.DELETE);
1650             }
1651             transactionObjects.add(subTransaction);
1652         }
1653
1654         if (addOrRemove == NwConstants.ADD_FLOW) {
1655             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1656         } else {
1657             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1658         }
1659
1660         if(!wrTxPresent ){
1661             tx.submit();
1662         }
1663     }
1664
1665     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1666     private Node buildDpnNode(BigInteger dpnId) {
1667         NodeId nodeId = new NodeId("openflow:" + dpnId);
1668         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1669
1670         return nodeDpn;
1671     }
1672
1673     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1674                                     int addOrRemove, WriteTransaction tx) {
1675         Boolean wrTxPresent = true;
1676         if (tx == null) {
1677             wrTxPresent = false;
1678             tx = dataBroker.newWriteOnlyTransaction();
1679         }
1680
1681         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1682         matches.add(new MatchInfo(MatchFieldType.eth_type,
1683                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1684         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1685
1686         // Install the flow entry in L3_LFIB_TABLE
1687         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1688
1689         FlowEntity flowEntity;
1690         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1691                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1692         Flow flow = flowEntity.getFlowBuilder().build();
1693         String flowId = flowEntity.getFlowId();
1694         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1695         Node nodeDpn = buildDpnNode(dpId);
1696         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1697                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1698                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1699
1700         if (addOrRemove == NwConstants.ADD_FLOW) {
1701             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1702         } else {
1703             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1704         }
1705         if(!wrTxPresent ){
1706             tx.submit();
1707         }
1708
1709         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1710                   dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1711     }
1712
1713     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1714                                       final String ipPrefixAddress) {
1715         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1716         try {
1717             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1718         } catch (NullPointerException e) {
1719             LOG.trace("", e);
1720         }
1721     }
1722
1723     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1724             final FutureCallback<List<Void>> callback) {
1725         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1726         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1727         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1728         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1729         if (!vrfTable.isPresent()) {
1730             LOG.warn("VRF Table not yet available for RD {}", rd);
1731             if (callback != null) {
1732                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1733                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1734                 Futures.addCallback(listenableFuture, callback);
1735             }
1736             return;
1737         }
1738         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1739         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1740                 new Callable<List<ListenableFuture<Void>>>() {
1741             @Override
1742             public List<ListenableFuture<Void>> call() throws Exception {
1743                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1744                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1745                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1746                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1747                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1748                         if (subnetRoute != null) {
1749                             long elanTag = subnetRoute.getElantag();
1750                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1751                             continue;
1752                         }
1753                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1754                         if (routerInt != null) {
1755                             LOG.trace( "Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1756                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1757                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(),
1758                                     new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1759                             continue;
1760                         }
1761                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
1762                             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1763                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1764                                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1765                                 if (lri.getDpnId().equals(dpnId)) {
1766                                     createLocalFibEntry(vpnId, rd, vrfEntry);
1767                                     continue;
1768                                 }
1769                             }
1770                         }
1771                         // Passing null as we don't know the dpn
1772                         // to which prefix is attached at this point
1773                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1774                     }
1775                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1776                     futures.add(tx.submit());
1777                 }
1778                 if (callback != null) {
1779                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1780                     Futures.addCallback(listenableFuture, callback);
1781                 }
1782                 return futures;
1783             }
1784         });
1785     }
1786
1787
1788     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1789                                             final String localNextHopIp, final String remoteNextHopIp) {
1790         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1791                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1792         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1793         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1794         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1795         if (vrfTable.isPresent()) {
1796             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1797             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1798                     new Callable<List<ListenableFuture<Void>>>() {
1799                         @Override
1800                         public List<ListenableFuture<Void>> call() throws Exception {
1801                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1802                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1803                                 WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1804                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1805                                     if(!vrfEntry.getNextHopAddressList().isEmpty()) {
1806                                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1807                                             if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1808                                                 LOG.trace(" creating remote FIB entry for prefix {} rd {}", vrfEntry.getDestPrefix(), rd);
1809                                                 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
1810                                             }
1811                                         }
1812                                     }
1813                                 }
1814                                 futures.add(writeCfgTxn.submit());
1815                             }
1816                             return futures;
1817                         }
1818                     });
1819         }
1820     }
1821
1822     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1823                                             final String localNextHopIp, final String remoteNextHopIp) {
1824         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1825                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1826         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1827         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1828         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1829         if (vrfTable.isPresent()) {
1830             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1831             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1832                     new Callable<List<ListenableFuture<Void>>>() {
1833                         @Override
1834                         public List<ListenableFuture<Void>> call() throws Exception {
1835                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1836                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1837                                 WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1838                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1839                                     // Handle Internal Routes only (i.e., STATIC for now)
1840                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
1841                                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1842                                         /* Ignore SubnetRoute entry */
1843                                         if (subnetRoute == null) {
1844                                             if(!vrfEntry.getNextHopAddressList().isEmpty()) {
1845                                                 if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1846                                                     LOG.trace(" creating remote FIB entry for prefix {} rd {} on Dpn {}", vrfEntry.getDestPrefix(), rd, dpnId);
1847                                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
1848                                                 }
1849                                             }
1850                                         }
1851                                     }
1852                                 }
1853                                 futures.add(writeCfgTxn.submit());
1854                             }
1855                             return futures;
1856                         }
1857                     });
1858         }
1859     }
1860
1861     public void manageRemoteRouteOnDPN(final boolean action,
1862                                        final BigInteger localDpnId,
1863                                        final long vpnId,
1864                                        final String  rd,
1865                                        final String destPrefix,
1866                                        final String destTepIp) {
1867         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1868
1869         if (vpnInstance == null) {
1870             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
1871             return;
1872         }
1873         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1874         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
1875                 new Callable<List<ListenableFuture<Void>>>() {
1876                     @Override
1877                     public List<ListenableFuture<Void>> call() throws Exception {
1878                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1879                         synchronized (vpnInstance.getVpnInstanceName().intern()) {
1880                             WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1881                             VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1882                             VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1883                             if (vrfEntry == null)
1884                                 return futures;
1885                             LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
1886                                     action, localDpnId, vpnId, rd, destPrefix);
1887                             List<String> nhList = new ArrayList<String>();
1888                             List<String> nextHopAddressList = vrfEntry.getNextHopAddressList();
1889                             VrfEntry modVrfEntry;
1890                             if (nextHopAddressList == null || (nextHopAddressList.isEmpty())) {
1891                                 nhList = Arrays.asList(destTepIp);
1892                                 modVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(nhList).build();
1893                             } else {
1894                                 modVrfEntry = vrfEntry;
1895                             }
1896
1897                             if (action == true) {
1898                                 LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
1899                                 createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1900                             } else {
1901                                 LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
1902                                 deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
1903                             }
1904                             futures.add(writeTransaction.submit());
1905                         }
1906                         return futures;
1907                     }
1908                 });
1909     }
1910
1911     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1912                                  final FutureCallback<List<Void>> callback) {
1913         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1914         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1915         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1916         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1917         if (vrfTable.isPresent()) {
1918             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1919             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1920                     new Callable<List<ListenableFuture<Void>>>() {
1921                         @Override
1922                         public List<ListenableFuture<Void>> call() throws Exception {
1923                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1924                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1925                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1926                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1927                                     /* Handle subnet routes here */
1928                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1929                                     if (subnetRoute != null) {
1930                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1931                                                 dpnId, rd);
1932                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1933                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1934                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1935                                                 vrfEntry.getDestPrefix());
1936                                         continue;
1937                                     }
1938                                     // ping responder for router interfaces
1939                                     RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1940                                     if (routerInt != null) {
1941                                         LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1942                                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1943                                         installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(),
1944                                                 new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
1945                                         continue;
1946                                     }
1947                                     // Passing null as we don't know the dpn
1948                                     // to which prefix is attached at this point
1949                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1950                                 }
1951                                 futures.add(tx.submit());
1952                                 if (callback != null) {
1953                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1954                                     Futures.addCallback(listenableFuture, callback);
1955                                 }
1956                             }
1957                             return futures;
1958                         }
1959                     });
1960         }
1961     }
1962
1963     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1964                                            final String localNextHopIp, final String remoteNextHopIp) {
1965         LOG.trace(  "cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1966                         " localNexthopIp {} , remoteNexhtHopIp {}",
1967                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1968         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1969         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1970         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1971         if (vrfTable.isPresent()) {
1972             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1973             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1974                     new Callable<List<ListenableFuture<Void>>>() {
1975                         @Override
1976                         public List<ListenableFuture<Void>> call() throws Exception {
1977                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1978                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1979                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1980                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1981                                     if(!vrfEntry.getNextHopAddressList().isEmpty()) {
1982                                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1983                                             if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1984                                                 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1985                                                 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1986                                             }
1987                                         }
1988                                     }
1989                                 }
1990                                 futures.add(writeTransaction.submit());
1991                             }
1992                             return futures;
1993                         }
1994                     });
1995
1996         }
1997     }
1998     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1999                                            final String localNextHopIp, final String remoteNextHopIp) {
2000         LOG.trace(  "cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, " +
2001                         " localNexthopIp {} , remoteNexhtHopIp {}",
2002                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2003         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2004         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2005         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2006         if (vrfTable.isPresent()) {
2007             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2008             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2009                     new Callable<List<ListenableFuture<Void>>>() {
2010                         @Override
2011                         public List<ListenableFuture<Void>> call() throws Exception {
2012                             List<ListenableFuture<Void>> futures = new ArrayList<>();
2013                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
2014                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2015                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2016                                     // Handle Internal Routes only (i.e, STATIC for now)
2017                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
2018                                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2019                                         /* Ignore SubnetRoute entry */
2020                                         if (subnetRoute == null) {
2021                                             if (!vrfEntry.getNextHopAddressList().isEmpty()) {
2022                                                 if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
2023                                                     LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2024                                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
2025                                                 }
2026                                             }
2027                                         }
2028                                     }
2029                                 }
2030                                 futures.add(writeTransaction.submit());
2031                             }
2032                             return futures;
2033                         }
2034                     });
2035         }
2036     }
2037
2038     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2039         InstanceIdentifierBuilder<VrfTables> idBuilder =
2040                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2041         InstanceIdentifier<VrfTables> id = idBuilder.build();
2042         return id;
2043     }
2044
2045     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2046         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2047                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
2048                 .append(priority).toString();
2049     }
2050
2051     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2052         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2053                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
2054                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
2055                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
2056                 .append(destPrefix.getHostAddress()).toString();
2057     }
2058
2059     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
2060         return new StringBuilder(64).append(FLOWID_PREFIX)
2061                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
2062                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
2063                 .append(nextHop).toString();
2064     }
2065
2066     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2067                                             final VrfEntry vrfEntry, String rd) {
2068         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2069         List<String> prefixIpList = new ArrayList<>();
2070         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2071                 remoteDpnId, vpnId, vrfEntry);
2072         try {
2073             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2074                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
2075                 if (extra_route == null) {
2076                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2077                 } else {
2078                     prefixIpList = new ArrayList<>();
2079                     for (String extraRouteIp : extra_route.getNexthopIpList()) {
2080                         prefixIpList.add(extraRouteIp + "/32");
2081                     }
2082                 }
2083             } else {
2084                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2085             }
2086
2087             for (String prefixIp : prefixIpList) {
2088                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
2089                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
2090                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2091                             prefixIp, nextHopIp);
2092                     if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2093                         adjacencyList.add(adjacencyResult);
2094                     }
2095                 }
2096             }
2097         } catch (NullPointerException e) {
2098             LOG.trace("", e);
2099         }
2100         return adjacencyList;
2101     }
2102
2103     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2104         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2105                 InstanceIdentifier.create(VpnInstanceOpData.class)
2106                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2107         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2108                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2109         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2110     }
2111
2112     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2113         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2114                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2115                 .append(FLOWID_PREFIX).toString();
2116     }
2117
2118     /*
2119      * Install flow entry in protocol table to forward mpls
2120      * coming through gre tunnel to LFIB table.
2121      */
2122     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2123         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2124         // Instruction to goto L3 InterfaceTable
2125         List<InstructionInfo> instructions = new ArrayList<>();
2126         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2127         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2128         matches.add(new MatchInfo(MatchFieldType.eth_type,
2129                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2130         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2131                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2132                         NwConstants.L3_LFIB_TABLE),
2133                 DEFAULT_FIB_FLOW_PRIORITY,
2134                 "Protocol Table For LFIB",
2135                 0, 0,
2136                 COOKIE_PROTOCOL_TABLE,
2137                 matches, instructions);
2138
2139         if (addOrRemove == NwConstants.ADD_FLOW) {
2140             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2141             mdsalManager.installFlow(flowEntityToLfib);
2142         } else {
2143             mdsalManager.removeFlow(flowEntityToLfib);
2144         }
2145     }
2146
2147     public List<String> printFibEntries() {
2148         List<String> result = new ArrayList<>();
2149         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2150         result.add("-------------------------------------------------------------------");
2151         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2152         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2153         if (fibEntries.isPresent()) {
2154             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2155             for (VrfTables vrfTable : vrfTables) {
2156                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2157                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2158                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2159                                 vrfTable.getRouteDistinguisher(),
2160                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2161                     }
2162                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2163                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2164                                 vrfTable.getRouteDistinguisher(),
2165                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2166                     }
2167                 }
2168             }
2169         }
2170         return result;
2171     }
2172
2173
2174     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2175         InstanceIdentifier<VrfEntry> vrfEntryId =
2176                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2177                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2178         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2179         if (vrfEntry.isPresent())  {
2180             return vrfEntry.get();
2181         }
2182         return null;
2183     }
2184
2185     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2186         InstanceIdentifier<VrfEntry> vrfEntryId =
2187                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2188                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2189         return vrfEntryId;
2190     }
2191
2192     protected Boolean isIpv4Address(String ipAddress) {
2193         try {
2194             InetAddress address = InetAddress.getByName(ipAddress);
2195             if (address instanceof Inet4Address) {
2196                 return true;
2197             }
2198         } catch (UnknownHostException e) {
2199             LOG.warn("Invalid ip address {}", ipAddress, e);
2200             return false;
2201         }
2202         return false;
2203     }
2204
2205     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2206             long vpnId, int addOrRemove) {
2207         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2208         if (routerInt != null && vpnToDpnList != null) {
2209             String routerId = routerInt.getUuid();
2210             String macAddress = routerInt.getMacAddress();
2211             String ipValue = routerInt.getIpAddress();
2212             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2213                     routerId, ipValue, macAddress);
2214             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2215                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2216                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2217                             new MacAddress(macAddress), addOrRemove);
2218                 }
2219             }
2220             return true;
2221         }
2222         return false;
2223     }
2224
2225     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2226                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2227         String[] subSplit = routerInternalIp.split("/");
2228         if (!isIpv4Address(subSplit[0])) {
2229             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2230             return;
2231         }
2232
2233         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2234         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2235                 dpnId, routerInternalIp, vpnId, subSplit[0]);
2236
2237         List<MatchInfo> matches = new ArrayList<>();
2238
2239         matches.add(new MatchInfo(MatchFieldType.ip_proto, new long[] { IPProtocols.ICMP.intValue() }));
2240         matches.add(new MatchInfo(MatchFieldType.metadata,
2241                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
2242         matches.add(new MatchInfo(MatchFieldType.icmp_v4, new long[] { (short) 8, (short) 0 }));
2243         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
2244         matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { subSplit[0], "32" }));
2245
2246         List<ActionInfo> actionsInfos = new ArrayList<>();
2247
2248         // Set Eth Src and Eth Dst
2249         actionsInfos.add(new ActionInfo(ActionType.move_src_dst_eth, new String[] {}));
2250         actionsInfos.add(new ActionInfo(ActionType.set_field_eth_src, new String[] { routerMac.getValue() }));
2251
2252         // Move Ip Src to Ip Dst
2253         actionsInfos.add(new ActionInfo(ActionType.move_src_dst_ip, new String[] {}));
2254         actionsInfos.add(new ActionInfo(ActionType.set_source_ip, new String[] { subSplit[0], "32" }));
2255
2256         // Set the ICMP type to 0 (echo reply)
2257         actionsInfos.add(new ActionInfo(ActionType.set_icmp_type, new String[] { "0" }));
2258
2259         actionsInfos.add(new ActionInfo(ActionType.nx_load_in_port, new BigInteger[]{ BigInteger.ZERO }));
2260
2261         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit,
2262                 new String[] { Short.toString(NwConstants.L3_FIB_TABLE) }));
2263
2264         List<InstructionInfo> instructions = new ArrayList<>();
2265
2266         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2267
2268         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2269         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
2270
2271         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2272                 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2273
2274         if (addOrRemove == NwConstants.ADD_FLOW) {
2275             mdsalManager.installFlow(flowEntity);
2276         } else {
2277             mdsalManager.removeFlow(flowEntity);
2278         }
2279     }
2280
2281     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2282                                              final boolean isVpnFirstEndPoint,
2283                                              final VrfEntry vrfEntry)
2284     {
2285         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
2286                                     && vrfEntry.getNextHopAddressList().size() == 1);
2287         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2288
2289         if ( !interVpnLinkState.isPresent()) {
2290             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2291             return;
2292         }
2293
2294         List<BigInteger> targetDpns =
2295             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2296             : interVpnLinkState.get().getSecondEndpointState().getDpId();
2297
2298         String nextHop = vrfEntry.getNextHopAddressList().get(0);
2299
2300         // delete from FIB
2301         //
2302         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2303         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2304         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
2305             .setFlowName(flowRef).build();
2306
2307         LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2308                   interVpnLinkName, flowRef);
2309
2310         for ( BigInteger dpId : targetDpns ) {
2311             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2312                       vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2313                       dpId, interVpnLinkName);
2314
2315             mdsalManager.removeFlow(dpId, flow);
2316         }
2317
2318         // delete from LFIB
2319         //
2320         LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2321
2322         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2323         for ( BigInteger dpId : targetDpns ) {
2324             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
2325                       vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2326                       dpId, interVpnLinkName);
2327             makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */,
2328                                LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2329         }
2330         tx.submit();
2331     }
2332 }