vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15
  16 import java.math.BigInteger;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import java.util.concurrent.LinkedBlockingQueue;
  28
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  34 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.ActionType;
  37 import org.opendaylight.genius.mdsalutil.FlowEntity;
  38 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  39 import org.opendaylight.genius.mdsalutil.InstructionType;
  40 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  41 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  42 import org.opendaylight.genius.mdsalutil.MatchInfo;
  43 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.packet.IPProtocols;
  47 import org.opendaylight.genius.utils.ServiceIndex;
  48 import org.opendaylight.genius.utils.batching.ActionableResource;
  49 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  50 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  51 import org.opendaylight.genius.utils.batching.ResourceHandler;
  52 import org.opendaylight.genius.utils.batching.SubTransaction;
  53 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  57 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  58 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 117 import org.opendaylight.yangtools.yang.binding.DataObject;
 118 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 120 import org.opendaylight.yangtools.yang.common.RpcResult;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> implements AutoCloseable, ResourceHandler {
 125     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 126     private static final String FLOWID_PREFIX = "L3.";
 127     private final DataBroker dataBroker;
 128     private final IMdsalApiManager mdsalManager;
 129     private IVpnManager vpnmanager;
 130     private final NexthopManager nextHopManager;
 131     private ItmRpcService itmManager;
 132     private final OdlInterfaceRpcService interfaceManager;
 133     private final IdManagerService idManager;
 134     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 135     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 136     private static final int LFIB_INTERVPN_PRIORITY = 1;
 137     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 138     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 139     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 140     List<SubTransaction> transactionObjects;
 141     private static final int PERIODICITY = 500;
 142     private static Integer batchSize;
 143     private static Integer batchInterval;
 144     private static final int BATCH_SIZE = 1000;
 145     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 146     private final ResourceBatchingManager resourceBatchingManager;
 147
 148     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 149                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 150                             final IdManagerService idManager) {
 151         super(VrfEntry.class, VrfEntryListener.class);
 152         this.dataBroker = dataBroker;
 153         this.mdsalManager = mdsalApiManager;
 154         this.nextHopManager = nexthopManager;
 155         this.interfaceManager = interfaceManager;
 156         this.idManager = idManager;
 157         batchSize = Integer.getInteger("batch.size");
 158         if (batchSize == null) {
 159             batchSize = BATCH_SIZE;
 160         }
 161         batchInterval = Integer.getInteger("batch.wait.time");
 162         if (batchInterval == null) {
 163             batchInterval = PERIODICITY;
 164         }
 165         resourceBatchingManager = ResourceBatchingManager.getInstance();
 166         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 167         transactionObjects = new ArrayList<>();
 168     }
 169
 170     public void start() {
 171         LOG.info("{} start", getClass().getSimpleName());
 172         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 173     }
 174
 175     @Override
 176     protected VrfEntryListener getDataTreeChangeListener() { return VrfEntryListener.this; }
 177
 178     @Override
 179     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 180         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 181     }
 182
 183     @Override
 184     public DataBroker getResourceBroker() {
 185         return dataBroker;
 186     }
 187
 188     @Override
 189     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 190         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 191         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 192         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 193                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 194         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 195             createFibEntries(identifier, vrfEntry);
 196         } else {
 197             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 198             actResource.setAction(ActionableResource.CREATE);
 199             actResource.setInstanceIdentifier(identifier);
 200             actResource.setInstance(vrfEntry);
 201             vrfEntryBufferQ.add(actResource);
 202             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 203         }
 204         LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 205                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 206     }
 207
 208     @Override
 209     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 210         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 211         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 212         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 213                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 214         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 215             deleteFibEntries(identifier, vrfEntry);
 216         } else {
 217             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 218             actResource.setAction(ActionableResource.DELETE);
 219             actResource.setInstanceIdentifier(identifier);
 220             actResource.setInstance(vrfEntry);
 221             vrfEntryBufferQ.add(actResource);
 222             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 223         }
 224         LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 225                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 226     }
 227
 228     @Override
 229     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 230         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 231         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 232         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 233                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 234         if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 235             createFibEntries(identifier, update);
 236         } else {
 237             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 238             actResource.setAction(ActionableResource.UPDATE);
 239             actResource.setInstanceIdentifier(identifier);
 240             actResource.setInstance(update);
 241             actResource.setOldInstance(original);
 242             vrfEntryBufferQ.add(actResource);
 243         }
 244         LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 245                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 246     }
 247
 248     @Override
 249     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
 250         this.transactionObjects = transactionObjects;
 251         if (vrfEntry instanceof VrfEntry) {
 252             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 253         }
 254     }
 255
 256     @Override
 257     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
 258         this.transactionObjects = transactionObjects;
 259         if (vrfEntry instanceof VrfEntry) {
 260             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 261         }
 262     }
 263
 264     @Override
 265     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 266                        Object update, List<SubTransaction> transactionObjects) {
 267         this.transactionObjects = transactionObjects;
 268         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 269             createFibEntries(tx, identifier, (VrfEntry)update);
 270         }
 271     }
 272
 273     @Override
 274     public int getBatchSize() {
 275         return batchSize;
 276     }
 277
 278     @Override
 279     public int getBatchInterval() {
 280         return batchInterval;
 281     }
 282
 283     @Override
 284     public LogicalDatastoreType getDatastoreType() {
 285         return LogicalDatastoreType.CONFIGURATION;
 286     }
 287
 288     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 289         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 290
 291         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 292         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 293         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 294
 295         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 296         final Long vpnId = vpnInstance.getVpnId();
 297         final String rd = vrfTableKey.getRouteDistinguisher();
 298         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 299         if (subnetRoute != null) {
 300             final long elanTag = subnetRoute.getElantag();
 301             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 302                     rd, vrfEntry.getDestPrefix(), elanTag);
 303             if (vpnToDpnList != null) {
 304                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 305                 dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 306                         new Callable<List<ListenableFuture<Void>>>() {
 307                             @Override
 308                             public List<ListenableFuture<Void>> call() throws Exception {
 309                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 310                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 311                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 312                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 313                                     }
 314                                 }
 315                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 316                                 futures.add(tx.submit());
 317                                 return futures;
 318                             }
 319                         });
 320             }
 321             return;
 322         }
 323         // ping responder for router interfaces
 324         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 325             return;
 326         }
 327
 328         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 329             // When it is a leaked route, the LFIB and FIB goes a bit different.
 330             installInterVpnRouteInLFib(rd, vrfEntry);
 331             return;
 332         }
 333
 334         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 335
 336         if (vpnToDpnList != null) {
 337             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 338             dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 339                     new Callable<List<ListenableFuture<Void>>>() {
 340                         @Override
 341                         public List<ListenableFuture<Void>> call() throws Exception {
 342                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 343                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 344                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 345                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 346                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 347                                     }
 348                                 }
 349                             }
 350                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 351                             futures.add(tx.submit());
 352                             return futures;
 353                         }
 354                     });
 355         }
 356
 357         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 358         if ( optVpnUuid.isPresent() ) {
 359             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 360             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 361             if ( optInterVpnLink.isPresent() ) {
 362                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 363                 String vpnUuid = optVpnUuid.get();
 364                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 365                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid) ) {
 366                     // This is an static route that points to the other endpoint of an InterVpnLink
 367                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 368                     installRouteInInterVpnLink(interVpnLink, vpnUuid, vrfEntry, vpnId);
 369                     installInterVpnRouteInLFib(rd, vrfEntry);
 370                 }
 371             }
 372         }
 373     }
 374
 375
 376     /*
 377       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 378       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 379       provided by ResourceBatchingManager
 380      */
 381     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 382         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 383
 384         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 385         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 386         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 387
 388         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 389         final String rd = vrfTableKey.getRouteDistinguisher();
 390         if (vpnToDpnList != null) {
 391             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 392                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 393                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 394                 }
 395             }
 396         }
 397     }
 398
 399     // FIXME: Refactoring needed here.
 400     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 401     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 402                                    int addOrRemove) {
 403         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 404         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 405
 406         String rd = vrfTableKey.getRouteDistinguisher();
 407         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 408         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 409             if (vpnInstance == null) {
 410                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 411                 return;
 412             }
 413         } else {
 414             Preconditions.checkNotNull(vpnInstance,
 415                                        "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 416         }
 417         String vpnUuid = vpnInstance.getVpnInstanceName();
 418         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 419                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 420
 421         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 422         // there an interVpnLink for the involved vpn in order to make learn the new route to
 423         // the other part of the inter-vpn-link.
 424
 425         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 426         Optional<InterVpnLink> interVpnLink =
 427             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 428                                                   : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 429         if ( !interVpnLink.isPresent() ) {
 430             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 431             return;
 432         }
 433
 434         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 435         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 436         boolean proceed =
 437             (addOrRemove == NwConstants.DEL_FLOW) || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 438                                                        && interVpnLink.get().isBgpRoutesLeaking() );
 439
 440         if ( proceed ) {
 441             String theOtherVpnId = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid)
 442                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 443                     : vpnUuid;
 444
 445             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 446             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 447
 448             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 449                     InstanceIdentifier.builder(FibEntries.class)
 450                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 451                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 452                             .build();
 453             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 454                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 455                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 456                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 457                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 458                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 459                         .setLabel(label)
 460                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 461                         .build();
 462                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 463             } else {
 464                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 465                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 466             }
 467         }
 468     }
 469
 470     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 471         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 472         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 473         prefixBuilder.setDpnId(lri.getDpnId());
 474         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 475         prefixBuilder.setIpAddress(lri.getPrefix());
 476         // Increment the refCount here
 477         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 478                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 479         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 480         if (!isPresentInList) {
 481             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 482             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 483             vpnInstanceNames.add(vpnInstanceName);
 484             builder.setVpnInstanceList(vpnInstanceNames);
 485             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 486         } else {
 487             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 488         }
 489         return prefixBuilder.build();
 490     }
 491
 492     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 493                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 494         Boolean wrTxPresent = true;
 495         if (tx == null) {
 496             wrTxPresent = false;
 497             tx = dataBroker.newWriteOnlyTransaction();
 498         }
 499         synchronized (vrfEntry.getLabel().toString().intern()) {
 500             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 501             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 502                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 503
 504                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 505                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 506                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 507                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 508                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 509                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 510                         }
 511                     }
 512                 }
 513                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 514                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 515             }
 516         }
 517         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 518         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 519         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 520         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 521         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 522
 523         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 524             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 525             // reinitialize instructions list for LFIB Table
 526             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 527
 528             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 529             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 530             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 531             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 532
 533             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 534         }
 535         if (!wrTxPresent ) {
 536             tx.submit();
 537         }
 538     }
 539
 540     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 541         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 542         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 543         // packet is commuted from Vpn2 to Vpn1.
 544         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 545         if ( !vpnNameOpc.isPresent() ) {
 546             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 547             return;
 548         }
 549
 550         String vpnName = vpnNameOpc.get();
 551         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 552         boolean interVpnLinkFound = false;
 553         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 554             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 555             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 556                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 557             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 558                 interVpnLinkFound = true;
 559
 560                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 561                 if ( !vpnLinkState.isPresent()
 562                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 563                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 564                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 565                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 566                     return;
 567                 }
 568
 569                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 570                                                                : vpnLinkState.get().getSecondEndpointState().getDpId();
 571                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 572                                                  : vpnLinkState.get().getFirstEndpointState().getLportTag();
 573
 574                 for ( BigInteger dpId : targetDpns ) {
 575                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 576
 577                     BigInteger[] metadata = new BigInteger[] {
 578                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 579                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 580                     };
 581                     List<InstructionInfo> instructions =
 582                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 583                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 584                                     new InstructionInfo(InstructionType.goto_table,
 585                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 586
 587                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 588                             NwConstants.ADD_FLOW, null);
 589                 }
 590
 591                 break;
 592             }
 593         }
 594
 595         if ( !interVpnLinkFound ) {
 596             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 597                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 598         }
 599     }
 600
 601
 602
 603     private void installRouteInInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 604                                             final VrfEntry vrfEntry, long vpnTag) {
 605         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 606         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 607                 && vrfEntry.getNextHopAddressList().size() == 1);
 608         String destination = vrfEntry.getDestPrefix();
 609         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 610         String iVpnLinkName = interVpnLink.getInterVpnLinkName();
 611
 612         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 613         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 614         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 615         if ( interVpnLink.getState().or(State.Error) != State.Active ) {
 616             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 617                     destination, nextHop, iVpnLinkName);
 618             return;
 619         }
 620
 621         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 622         if ( !optOtherEndpointLportTag.isPresent() ) {
 623             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 624                      vpnUuid, iVpnLinkName);
 625             return;
 626         }
 627
 628         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 629         if ( targetDpns.isEmpty() ) {
 630             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, iVpnLinkName);
 631             return;
 632         }
 633
 634         BigInteger[] metadata = new BigInteger[] {
 635                 MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 636                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 637                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 638         };
 639         List<Instruction> instructions =
 640                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 641                               new InstructionInfo(InstructionType.goto_table,
 642                                                   new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 643
 644         String values[] = destination.split("/");
 645         String destPrefixIpAddress = values[0];
 646         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 647
 648         List<MatchInfo> matches = new ArrayList<>();
 649         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 650                                                                               MetaDataUtil.METADATA_MASK_VRFID }));
 651         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 652
 653         if (prefixLength != 0) {
 654             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 655                                       new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 656         }
 657
 658         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 659         String flowRef = getInterVpnFibFlowRef(iVpnLinkName, destination, nextHop);
 660         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 661                                                  COOKIE_VM_FIB_TABLE, matches, instructions);
 662
 663         for ( BigInteger dpId : targetDpns ) {
 664             mdsalManager.installFlow(dpId, flowEntity);
 665         }
 666     }
 667
 668     private void removeRouteFromInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 669                                              final VrfEntry vrfEntry) {
 670
 671         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 672         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 673                                     && vrfEntry.getNextHopAddressList().size() == 1);
 674
 675         String iVpnLinkName = interVpnLink.getInterVpnLinkName();
 676
 677         InterVpnLinkState interVpnLinkState = interVpnLink.getInterVpnLinkState();
 678         if ( interVpnLinkState == null ) {
 679             LOG.warn("Could not find State for InterVpnLink {}", iVpnLinkName);
 680             return;
 681         }
 682
 683         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 684         String flowRef = getInterVpnFibFlowRef(iVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
 685         FlowId flowId = new FlowId(flowRef);
 686         Flow flow = new FlowBuilder().setKey(new FlowKey(flowId)).setId(flowId).setTableId(NwConstants.L3_FIB_TABLE)
 687                                      .setFlowName(flowRef).build();
 688
 689         for ( BigInteger dpId : interVpnLink.getEndpointDpnsByVpnName(vpnUuid) ) {
 690             mdsalManager.removeFlow(dpId, flow);
 691         }
 692
 693     }
 694
 695     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 696                                                      InstanceIdentifier<T> path) {
 697
 698         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 699
 700         Optional<T> result = Optional.absent();
 701         try {
 702             result = tx.read(datastoreType, path).get();
 703         } catch (Exception e) {
 704             throw new RuntimeException(e);
 705         }
 706
 707         return result;
 708     }
 709
 710     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 711         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 712         Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 713         String localNextHopIP = vrfEntry.getDestPrefix();
 714
 715         if (localNextHopInfo == null) {
 716             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 717             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 718             if (extraRoute != null) {
 719                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 720                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 721                     if (nextHopIp != null) {
 722                         localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
 723                         localNextHopIP = nextHopIp + "/32";
 724                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 725                         returnLocalDpnId.add(dpnId);
 726                     }
 727                 }
 728             }
 729             if (localNextHopInfo == null) {
 730             /* imported routes case */
 731                 synchronized (vrfEntry.getLabel().toString().intern()) {
 732                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 733                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 734                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 735                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 736                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 737                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 738                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 739                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 740                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 741                                     localNextHopIP = lri.getPrefix();
 742                                 } else {
 743                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 744                                     localNextHopIP = lri.getPrefix();
 745                                 }
 746                             }
 747                             if (localNextHopInfo != null) {
 748                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 749                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 750                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 751                                 returnLocalDpnId.add(dpnId);
 752                             }
 753                         }
 754                     }
 755                 }
 756             }
 757         } else {
 758             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 759             returnLocalDpnId.add(dpnId);
 760         }
 761
 762         return returnLocalDpnId;
 763     }
 764
 765     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 766                                                 final VrfEntry vrfEntry, Long parentVpnId){
 767         if (localNextHopInfo != null) {
 768             final BigInteger dpnId = localNextHopInfo.getDpnId();
 769             if (!isVpnPresentInDpn(rd, dpnId)) {
 770                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 771                 return BigInteger.ZERO;
 772             }
 773
 774             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 775                     localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix());
 776             if (groupId == 0) {
 777                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 778                         vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 779                 return BigInteger.ZERO;
 780             }
 781             List<ActionInfo> actionsInfos =
 782                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 783             final List<InstructionInfo> instructions =
 784                     Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 785             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 786                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 787             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 788             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 789                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 790                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 791             } else {
 792                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 793             }
 794             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 795             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 796                     new Callable<List<ListenableFuture<Void>>>() {
 797                         @Override
 798                         public List<ListenableFuture<Void>> call() throws Exception {
 799                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 800                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 801                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 802                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 803                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 804                             }
 805                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 806                             futures.add(tx.submit());
 807                             return futures;
 808                         }
 809                     });
 810             return dpnId;
 811         }
 812         return BigInteger.ZERO;
 813     }
 814
 815     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 816         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 817         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 818         if (dpnInVpn.isPresent()) {
 819             return true;
 820         }
 821         return false;
 822     }
 823
 824     private LabelRouteInfo getLabelRouteInfo(Long label) {
 825         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 826                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 827         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 828         if (opResult.isPresent()) {
 829             return opResult.get();
 830         }
 831         return null;
 832     }
 833
 834     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 835         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 836         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 837                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 838         if (lri == null) {
 839             return true;
 840         }
 841         List<String> vpnInstancesList = lri.getVpnInstanceList() != null ? lri.getVpnInstanceList() : new ArrayList<String>();
 842         if (vpnInstancesList.contains(vpnInstanceName)) {
 843             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 844             vpnInstancesList.remove(vpnInstanceName);
 845         }
 846         if (vpnInstancesList.size() == 0) {
 847             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 848             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 849             return true;
 850         } else {
 851             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 852             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 853             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 854         }
 855         return false;
 856     }
 857
 858     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 859                                       WriteTransaction tx) {
 860         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 861         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 862
 863
 864         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 865
 866         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 867                 dpId, label, groupId);
 868     }
 869
 870     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 871                                                  WriteTransaction tx) {
 872         List<MatchInfo> mkMatches = new ArrayList<>();
 873
 874         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 875
 876         // Matching metadata
 877         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 878         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 879
 880         List<InstructionInfo> mkInstructions = new ArrayList<>();
 881         mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 882
 883         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 884                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 885                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 886
 887         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 888
 889         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 890
 891         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 892         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 893                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 894                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 895         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 896     }
 897
 898     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 899         FlowEntity flowEntity;
 900         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 901         List<MatchInfo> mkMatches = new ArrayList<>();
 902         // Matching metadata
 903         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 904         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 905                 NwConstants.INTERNAL_TUNNEL_TABLE,
 906                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 907                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 908                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 909         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 910         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 911         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 912                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 913                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 914
 915         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 916         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 917     }
 918
 919     /**
 920      * Delete local FIB entry
 921      * @param vpnId
 922      * @param rd
 923      * @param vrfEntry
 924      * @return
 925      */
 926     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 927         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 928         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 929         String localNextHopIP = vrfEntry.getDestPrefix();
 930
 931         if (localNextHopInfo == null) {
 932             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 933             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 934             if (extra_route != null) {
 935                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 936                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 937                     if (nextHopIp != null) {
 938                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 939                         localNextHopIP = nextHopIp + "/32";
 940                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 941                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 942                         if (!dpnId.equals(BigInteger.ZERO)) {
 943                             returnLocalDpnId.add(dpnId);
 944                         }
 945                     }
 946                 }
 947             }
 948
 949             if (localNextHopInfo == null) {
 950               /* Imported VRF entry */
 951                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 952                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 953                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 954                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
 955                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
 956                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
 957                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 958                     if (!dpnId.equals(BigInteger.ZERO)) {
 959                         returnLocalDpnId.add(dpnId);
 960                     }
 961                 }
 962             }
 963
 964
 965         } else {
 966             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 967                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 968             if (!dpnId.equals(BigInteger.ZERO)) {
 969                 returnLocalDpnId.add(dpnId);
 970             }
 971         }
 972
 973         return returnLocalDpnId;
 974     }
 975
 976     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
 977                                                 final Long vpnId, final String rd,
 978                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
 979         if (localNextHopInfo != null) {
 980             final BigInteger dpnId = localNextHopInfo.getDpnId();;
 981             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 982             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 983                     new Callable<List<ListenableFuture<Void>>>() {
 984                         @Override
 985                         public List<ListenableFuture<Void>> call() throws Exception {
 986                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 987                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
 988                                     NwConstants.DEL_FLOW, tx);
 989                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 990                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
 991                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
 992                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
 993                             }
 994                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 995                             futures.add(tx.submit());
 996                             return futures;
 997                         }
 998                     });
 999             //TODO: verify below adjacency call need to be optimized (?)
1000             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1001             return dpnId;
1002         }
1003         return BigInteger.ZERO;
1004     }
1005
1006
1007     private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
1008         return InstanceIdentifier.builder(PrefixToInterface.class)
1009                 .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
1010     }
1011
1012     private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
1013         Optional<Prefixes> localNextHopInfoData =
1014                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
1015         return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
1016     }
1017
1018     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1019         return InstanceIdentifier.builder(VpnToExtraroute.class)
1020                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1021                         new ExtrarouteKey(ipPrefix)).build();
1022     }
1023
1024     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1025         Optional<Extraroute> extraRouteInfo =
1026                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1027         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1028
1029     }
1030
1031     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1032         try {
1033             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1034                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1035             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1036             if(!rpcResult.isSuccessful()) {
1037                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1038             } else {
1039                 return rpcResult.getResult().getTunnelType();
1040             }
1041
1042         } catch (InterruptedException | ExecutionException e) {
1043             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1044         }
1045
1046         return null;
1047
1048     }
1049     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1050                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1051         Boolean wrTxPresent = true;
1052         if (tx == null) {
1053             wrTxPresent = false;
1054             tx = dataBroker.newWriteOnlyTransaction();
1055         }
1056         String rd = vrfTableKey.getRouteDistinguisher();
1057         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1058                 vrfEntry.getDestPrefix(), rd, tx);
1059         /********************************************/
1060         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1061
1062         if (tunnelInterfaceList.isEmpty()) {
1063             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1064                     vrfEntry.getNextHopAddressList(), rd);
1065             LOG.warn("Failed to add Route: {} in vpn: {}",
1066                     vrfEntry.getDestPrefix(), rd);
1067             return;
1068         }
1069
1070         for (String tunnelInterface : tunnelInterfaceList) {
1071             List<InstructionInfo> instructions = new ArrayList<>();
1072             List<ActionInfo> actionInfos = new ArrayList<>();
1073             Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1074             if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1075                 LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1076                 actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
1077                 actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
1078             } else {
1079                 int label = vrfEntry.getLabel().intValue();
1080                 BigInteger tunnelId;
1081                 // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1082                 if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1083                     tunnelId = BigInteger.valueOf(label);
1084                 } else {
1085                     tunnelId = BigInteger.valueOf(label);
1086                 }
1087
1088                 LOG.debug("adding set tunnel id action for label {}", label);
1089                 actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
1090             }
1091             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
1092             if(egressActions.isEmpty()){
1093                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
1094                 return;
1095             }
1096             actionInfos.addAll(egressActions);
1097             instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
1098             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1099         }
1100         if(!wrTxPresent ){
1101             tx.submit();
1102         }
1103         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1104     }
1105
1106     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1107         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1108         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1109         if (dpnInVpn.isPresent()) {
1110             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1111             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1112
1113             if (vpnInterfaces.remove(currVpnInterface)) {
1114                 if (vpnInterfaces.isEmpty()) {
1115                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1116                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1117                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1118                 } else {
1119                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1120                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1121                             VpnInterfaces.class,
1122                             new VpnInterfacesKey(intfName)));
1123                 }
1124             }
1125         }
1126     }
1127
1128     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1129     /* Get interface info from prefix to interface mapping;
1130         Use the interface info to get the corresponding vpn interface op DS entry,
1131         remove the adjacency corresponding to this fib entry.
1132         If adjacency removed is the last adjacency, clean up the following:
1133          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1134          - prefix to interface entry
1135          - vpn interface op DS
1136      */
1137         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1138         Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1139         Extraroute extraRoute = null;
1140         if (prefixInfo == null) {
1141             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1142             if(extraRoute != null) {
1143                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1144                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1145
1146                     if (nextHopIp != null) {
1147                         prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
1148                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1149                     }
1150                 }
1151             }
1152             if (prefixInfo == null) {
1153                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1154                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1155                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1156                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1157                     prefixBuilder.setDpnId(lri.getDpnId());
1158                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1159                     prefixBuilder.setIpAddress(lri.getPrefix());
1160                     prefixInfo = prefixBuilder.build();
1161                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1162                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1163                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1164                 }
1165             }
1166         } else {
1167             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1168         }
1169     }
1170
1171     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1172                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1173
1174         if (prefixInfo == null) {
1175             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1176             return; //Don't have any info for this prefix (shouldn't happen); need to return
1177         }
1178
1179         String ifName = prefixInfo.getVpnInterfaceName();
1180         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1181         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1182                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1183     }
1184
1185     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1186         Prefixes prefixInfo;
1187         Long vpnId;
1188         String rd;
1189         VrfEntry vrfEntry;
1190         Extraroute extraRoute;
1191
1192         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1193                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1194             this.prefixInfo = prefixInfo;
1195             this.vpnId = vpnId;
1196             this.rd= rd;
1197             this.vrfEntry= vrfEntry;
1198             this.extraRoute = extraRoute;
1199         }
1200
1201         @Override
1202         public List<ListenableFuture<Void>> call() throws Exception {
1203             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1204             // to call the respective helpers.
1205
1206             //First Cleanup LabelRouteInfo
1207             synchronized (vrfEntry.getLabel().toString().intern()) {
1208                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1209                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1210                                 vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1211                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1212                     String vpnInstanceName = "";
1213                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1214                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1215                         }
1216                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1217                     if (lriRemoved) {
1218                             String parentRd = lri.getParentVpnRd();
1219                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1220                                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1221                         }
1222                 } else {
1223                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1224                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1225                 }
1226             }
1227             String ifName = prefixInfo.getVpnInterfaceName();
1228             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1229                     FibUtil.getVpnInterfaceIdentifier(ifName));
1230             if (optvpnInterface.isPresent()) {
1231                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1232                 if (vpnId != associatedVpnId) {
1233                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1234                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1235                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1236                     return null;
1237                 } else {
1238                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1239                             vrfEntry.getDestPrefix(), associatedVpnId);
1240                 }
1241             }
1242             if (extraRoute != null) {
1243                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1244                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1245             }
1246             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1247                     FibUtil.getAdjListPath(ifName));
1248             int numAdj = 0;
1249             if (optAdjacencies.isPresent()) {
1250                 numAdj = optAdjacencies.get().getAdjacency().size();
1251             }
1252             //remove adjacency corr to prefix
1253             if (numAdj > 1) {
1254                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1255                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1256                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1257             }
1258             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1259                 //clean up the vpn interface from DpnToVpn list
1260                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1261                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1262                         FibUtil.getVpnInterfaceIdentifier(ifName));
1263             }
1264             return null;
1265         }
1266     }
1267
1268     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1269         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1270         final String rd  = vrfTableKey.getRouteDistinguisher();
1271         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1272         if (vpnInstance == null) {
1273             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1274             return;
1275         }
1276         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1277         long elanTag = 0L;
1278         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1279         if (subnetRoute != null) {
1280             elanTag = subnetRoute.getElantag();
1281             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1282                     rd, vrfEntry.getDestPrefix(), elanTag);
1283             if (vpnToDpnList != null) {
1284                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1285                 dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1286                         new Callable<List<ListenableFuture<Void>>>() {
1287                             @Override
1288                             public List<ListenableFuture<Void>> call() throws Exception {
1289                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1290
1291                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1292
1293                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1294                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1295                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1296                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1297                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1298                                     }
1299                                 }
1300                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1301                                 futures.add(tx.submit());
1302                                 return futures;
1303                             }
1304                         });
1305             }
1306             synchronized (vrfEntry.getLabel().toString().intern()) {
1307                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1308                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1309                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1310                     String vpnInstanceName = "";
1311                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1312                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1313                     }
1314                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1315                     if (lriRemoved) {
1316                         String parentRd = lri.getParentVpnRd();
1317                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1318                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1319                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1320                                 vrfEntry.getDestPrefix());
1321                     }
1322                 } else {
1323                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1324                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1325                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1326                             vrfEntry.getDestPrefix());
1327                 }
1328             }
1329             return;
1330         }
1331         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1332             return;
1333         }
1334
1335         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1336                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1337         if (vpnToDpnList != null) {
1338             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1339             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1340                     new Callable<List<ListenableFuture<Void>>>() {
1341                         @Override
1342                         public List<ListenableFuture<Void>> call() throws Exception {
1343                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1344
1345                             if (localDpnIdList.size() <= 0) {
1346                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1347                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1348                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1349                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1350                                         }
1351                                     } else {
1352                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1353                                     }
1354                                 }
1355                             } else {
1356                                 for (BigInteger localDpnId : localDpnIdList) {
1357                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1358                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1359                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1360                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1361                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1362                                                 }
1363                                             } else {
1364                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1365                                             }
1366                                         }
1367                                     }
1368                                 }
1369                             }
1370                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1371                             futures.add(tx.submit());
1372                             return futures;
1373                         }
1374                     });
1375         }
1376
1377         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1378         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1379         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1380
1381         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1382         // of the interVpnLink.
1383         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1384         if ( optVpnUuid.isPresent() ) {
1385             String vpnUuid = optVpnUuid.get();
1386             List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
1387             if(routeNexthoplist.isEmpty()) {
1388                 LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
1389                 return;
1390             }
1391             String routeNexthop = routeNexthoplist.get(0);
1392             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1393             if ( optInterVpnLink.isPresent() ) {
1394                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1395                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid))
1396                 {
1397                     // This is route that points to the other endpoint of an InterVpnLink
1398                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1399                     removeRouteFromInterVpnLink(interVpnLink, rd, vrfEntry);
1400                 }
1401             }
1402         }
1403
1404     }
1405
1406     /*
1407       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1408       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1409       provided by ResourceBatchingManager
1410      */
1411     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1412         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1413
1414         final String rd  = vrfTableKey.getRouteDistinguisher();
1415         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1416         if (vpnInstance == null) {
1417             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1418             return;
1419         }
1420         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1421         if (vpnToDpnList != null) {
1422             for (VpnToDpnList curDpn : vpnToDpnList) {
1423                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1424                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1425                 }
1426             }
1427         }
1428     }
1429
1430     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1431                                   final long vpnId, final VrfTablesKey vrfTableKey,
1432                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1433
1434         Boolean wrTxPresent = true;
1435         if (tx == null) {
1436             wrTxPresent = false;
1437             tx = dataBroker.newWriteOnlyTransaction();
1438         }
1439
1440         LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1441         String rd = vrfTableKey.getRouteDistinguisher();
1442
1443         if(localDpnId != null) {
1444             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1445             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1446             return;
1447         }
1448
1449         // below two reads are kept as is, until best way is found to identify dpnID
1450         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1451         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1452
1453         if (localNextHopInfo == null && extraRoute != null) {
1454             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1455             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1456                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1457                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1458             }
1459         } else {
1460             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1461         }
1462         if(!wrTxPresent ){
1463             tx.submit();
1464         }
1465     }
1466
1467     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1468                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1469         boolean isRemoteRoute = true;
1470         if (localNextHopInfo != null) {
1471             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1472         }
1473         if (isRemoteRoute) {
1474             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1475             return true;
1476         } else {
1477             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1478             return false;
1479         }
1480     }
1481
1482     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1483         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1484         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1485     }
1486
1487     private long get
1488             (byte[] rawIpAddress) {
1489         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1490                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1491     }
1492
1493     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1494                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1495         Boolean wrTxPresent = true;
1496         if (tx == null) {
1497             wrTxPresent = false;
1498             tx = dataBroker.newWriteOnlyTransaction();
1499         }
1500
1501         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1502         String values[] = vrfEntry.getDestPrefix().split("/");
1503         String ipAddress = values[0];
1504         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1505         if (addOrRemove == NwConstants.ADD_FLOW) {
1506             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1507         } else {
1508             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1509         }
1510         InetAddress destPrefix;
1511         try {
1512             destPrefix = InetAddress.getByName(ipAddress);
1513         } catch (UnknownHostException e) {
1514             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1515             return;
1516         }
1517
1518         List<MatchInfo> matches = new ArrayList<>();
1519
1520         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1521                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1522
1523         matches.add(new MatchInfo(MatchFieldType.eth_type,
1524                 new long[] { NwConstants.ETHTYPE_IPV4 }));
1525
1526         if(prefixLength != 0) {
1527             matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1528                     destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1529         }
1530         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1531         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1532         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1533                 COOKIE_VM_FIB_TABLE, matches, instructions);
1534
1535         Flow flow = flowEntity.getFlowBuilder().build();
1536         String flowId = flowEntity.getFlowId();
1537         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1538         Node nodeDpn = buildDpnNode(dpId);
1539
1540         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1541                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1542                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1543
1544         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1545             SubTransaction subTransaction = new SubTransactionImpl();
1546             if (addOrRemove == NwConstants.ADD_FLOW) {
1547                 subTransaction.setInstanceIdentifier(flowInstanceId);
1548                 subTransaction.setInstance(flow);
1549                 subTransaction.setAction(SubTransaction.CREATE);
1550             } else {
1551                 subTransaction.setInstanceIdentifier(flowInstanceId);
1552                 subTransaction.setAction(SubTransaction.DELETE);
1553             }
1554             transactionObjects.add(subTransaction);
1555         }
1556
1557         if (addOrRemove == NwConstants.ADD_FLOW) {
1558             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1559         } else {
1560             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1561         }
1562
1563         if(!wrTxPresent ){
1564             tx.submit();
1565         }
1566     }
1567
1568     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1569     private Node buildDpnNode(BigInteger dpnId) {
1570         NodeId nodeId = new NodeId("openflow:" + dpnId);
1571         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1572
1573         return nodeDpn;
1574     }
1575
1576     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1577                                     int addOrRemove, WriteTransaction tx) {
1578         Boolean wrTxPresent = true;
1579         if (tx == null) {
1580             wrTxPresent = false;
1581             tx = dataBroker.newWriteOnlyTransaction();
1582         }
1583
1584         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1585         matches.add(new MatchInfo(MatchFieldType.eth_type,
1586                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1587         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1588
1589         // Install the flow entry in L3_LFIB_TABLE
1590         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1591
1592         FlowEntity flowEntity;
1593         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1594                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1595         Flow flow = flowEntity.getFlowBuilder().build();
1596         String flowId = flowEntity.getFlowId();
1597         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1598         Node nodeDpn = buildDpnNode(dpId);
1599         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1600                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1601                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1602
1603         if (addOrRemove == NwConstants.ADD_FLOW) {
1604             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1605         } else {
1606             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1607         }
1608         if(!wrTxPresent ){
1609             tx.submit();
1610         }
1611         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
1612                 dpId, label, instructions );
1613     }
1614
1615     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1616                                       final String ipPrefixAddress) {
1617         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1618         try {
1619             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1620         } catch (NullPointerException e) {
1621             LOG.trace("", e);
1622         }
1623     }
1624
1625     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1626             final FutureCallback<List<Void>> callback) {
1627         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1628         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1629         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1630         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1631         if (!vrfTable.isPresent()) {
1632             LOG.warn("VRF Table not yet available for RD {}", rd);
1633             if (callback != null) {
1634                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1635                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1636                 Futures.addCallback(listenableFuture, callback);
1637             }
1638             return;
1639         }
1640         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1641         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1642                 new Callable<List<ListenableFuture<Void>>>() {
1643             @Override
1644             public List<ListenableFuture<Void>> call() throws Exception {
1645                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1646                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1647                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1648                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1649                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1650                         if (subnetRoute != null) {
1651                             long elanTag = subnetRoute.getElantag();
1652                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1653                             continue;
1654                         }
1655                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1656                         if (routerInt != null) {
1657                             LOG.trace( "Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1658                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1659                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(),
1660                                     new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1661                             continue;
1662                         }
1663                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
1664                             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1665                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1666                                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1667                                 if (lri.getDpnId().equals(dpnId)) {
1668                                     createLocalFibEntry(vpnId, rd, vrfEntry);
1669                                     continue;
1670                                 }
1671                             }
1672                         }
1673                         // Passing null as we don't know the dpn
1674                         // to which prefix is attached at this point
1675                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1676                     }
1677                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1678                     futures.add(tx.submit());
1679                 }
1680                 if (callback != null) {
1681                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1682                     Futures.addCallback(listenableFuture, callback);
1683                 }
1684                 return futures;
1685             }
1686         });
1687     }
1688
1689
1690     public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1691                                  final String localNextHopIp, final String remoteNextHopIp) {
1692         LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1693                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1694         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1695         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1696         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1697         if (vrfTable.isPresent()) {
1698             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1699             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1700                     new Callable<List<ListenableFuture<Void>>>() {
1701                         @Override
1702                         public List<ListenableFuture<Void>> call() throws Exception {
1703                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1704                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1705                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1706                                 LOG.trace("populate FIB starts on Dpn " + dpnId
1707                                         + "rd  " + rd.toString()
1708                                         + "localNextHopIp " + localNextHopIp
1709                                         + "remoteNextHopIp" + remoteNextHopIp
1710                                         + "vpnId " + vpnId );
1711                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1712                                     LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1713
1714                                     if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1715                                         if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1716                                             LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1717                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1718                                         }
1719                                     } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
1720                                         BigInteger dpnIdForPrefix = null;
1721                                         String destPfx = vrfEntry.getDestPrefix();
1722                                         if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
1723                                             Optional<Extraroute> extraRouteInfo =
1724                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1725                                                             getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1726                                             if (extraRouteInfo.isPresent()) {
1727                                                 continue;
1728                                             }
1729                                             dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1730                                         } else {
1731                                             // Subnet Route handling
1732                                             Optional<Prefixes> localNextHopInfoData =
1733                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1734                                                             FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
1735                                             if (localNextHopInfoData.isPresent()) {
1736                                                 Prefixes prefixes = localNextHopInfoData.get();
1737                                                 dpnIdForPrefix = prefixes.getDpnId();
1738                                             }
1739                                         }
1740                                         if (dpnIdForPrefix == null) {
1741                                             LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1742                                                     vrfEntry.getDestPrefix());
1743                                             continue;
1744                                         }
1745                                         int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1746                                         if (sameDpnId != 0) {
1747                                             LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1748                                                     dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1749                                             continue;
1750                                         }
1751                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1752                                         List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
1753                                         newNextHopAddrList.add(localNextHopIp);
1754                                         VrfEntry newVrfEntry =
1755                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
1756                                         // Just update the VrfEntry
1757                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1758                                                 vrfEntryId, newVrfEntry);
1759                                         // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
1760                                         //       vrfEntryId, newVrfEntry);
1761                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1762                                         LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1763                                     }
1764                                 }
1765                                 futures.add(writeTransaction.submit());
1766                                 LOG.trace("populate FIB ends on Dpn " + dpnId
1767                                         + "rd  " + rd.toString()
1768                                         + "localNextHopIp " + localNextHopIp
1769                                         + "remoteNextHopIp" + remoteNextHopIp
1770                                         + "vpnId " + vpnId);
1771                             }
1772                             return futures;
1773                         }
1774                     });
1775         }
1776     }
1777
1778     public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
1779                                   final long vpnId, final String  rd, final String destPrefix ,
1780                                   final String localNextHopIP, final String remoteNextHopIp) {
1781
1782         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1783         dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1784                         + "local dpid" + localDpnId
1785                         + "remote dpid" + remoteDpnId
1786                         + "vpnId" + vpnId
1787                         + "localNHIp" + localNextHopIP
1788                         + "remoteNHIp" + remoteNextHopIp,
1789                 new Callable<List<ListenableFuture<Void>>>() {
1790                     @Override
1791                     public List<ListenableFuture<Void>> call() throws Exception {
1792                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1793                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1794                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1795                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1796                         if (vrfEntry == null) {
1797                             return futures;
1798                         }
1799                         LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1800                                         "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1801                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1802                         if (action == true) {
1803                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1804                             LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1805                             createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1806                         } else {
1807                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1808                             LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1809                             deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1810                         }
1811                         futures.add(writeTransaction.submit());
1812                         return futures;
1813                     }
1814                 });
1815     }
1816
1817     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1818                                  final FutureCallback<List<Void>> callback) {
1819         LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1820         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1821         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1822         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1823         if (vrfTable.isPresent()) {
1824             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1825             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1826                     new Callable<List<ListenableFuture<Void>>>() {
1827                         @Override
1828                         public List<ListenableFuture<Void>> call() throws Exception {
1829                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1830                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1831                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1832                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1833                                     /* Handle subnet routes here */
1834                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1835                                     if (subnetRoute != null) {
1836                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1837                                                 dpnId, rd);
1838                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1839                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1840                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1841                                                 vrfEntry.getDestPrefix());
1842                                         continue;
1843                                     }
1844                                     // ping responder for router interfaces
1845                                     RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1846                                     if (routerInt != null) {
1847                                         LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1848                                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1849                                         installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(),
1850                                                 new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
1851                                         continue;
1852                                     }
1853                                     // Passing null as we don't know the dpn
1854                                     // to which prefix is attached at this point
1855                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1856                                 }
1857                                 futures.add(tx.submit());
1858                                 if (callback != null) {
1859                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1860                                     Futures.addCallback(listenableFuture, callback);
1861                                 }
1862                             }
1863                             return futures;
1864                         }
1865
1866                     });
1867         }
1868     }
1869
1870     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1871                                  final String localNextHopIp, final String remoteNextHopIp,
1872                                  final FutureCallback<List<Void>> callback) {
1873         LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1874                         " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1875                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1876         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1877         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1878         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1879         if (vrfTable.isPresent()) {
1880             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1881             dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
1882                     new Callable<List<ListenableFuture<Void>>>() {
1883                         @Override
1884                         public List<ListenableFuture<Void>> call() throws Exception {
1885                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1886                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1887                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1888                                 LOG.trace("cleanup FIB starts on Dpn " + dpnId
1889                                         + "rd  " + rd.toString()
1890                                         + "localNextHopIp " + localNextHopIp
1891                                         + "remoteNextHopIp" + remoteNextHopIp
1892                                         + "vpnId " + vpnId);
1893
1894                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1895                                     LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1896                                     if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1897                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1898                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1899                                     }
1900
1901                                     if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1902                                         LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1903                                                 vrfEntry.getDestPrefix(), dpnId);
1904                                         String destPfx = vrfEntry.getDestPrefix();
1905                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1906                                         List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1907                                         newList.remove(localNextHopIp);
1908                                         VrfEntry newVrfEntry =
1909                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1910                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1911                                                 vrfEntryId, newVrfEntry);
1912                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1913                                         LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1914                                     }
1915                                 }
1916                                 futures.add(writeTransaction.submit());
1917                                 if (callback != null) {
1918                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1919                                     Futures.addCallback(listenableFuture, callback);
1920                                 }
1921                                 LOG.trace("cleanup FIB ends on Dpn " + dpnId
1922                                         + "rd  " + rd.toString()
1923                                         + "localNextHopIp " + localNextHopIp
1924                                         + "remoteNextHopIp" + remoteNextHopIp
1925                                         + "vpnId " + vpnId);
1926                             }
1927                             return futures;
1928                         }
1929                     });
1930
1931         }
1932     }
1933
1934     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1935         InstanceIdentifierBuilder<VrfTables> idBuilder =
1936                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1937         InstanceIdentifier<VrfTables> id = idBuilder.build();
1938         return id;
1939     }
1940
1941     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1942         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1943                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1944                 .append(priority).toString();
1945     }
1946
1947     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1948         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1949                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1950                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1951                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1952                 .append(destPrefix.getHostAddress()).toString();
1953     }
1954
1955     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1956         return new StringBuilder(64).append(FLOWID_PREFIX)
1957                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1958                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1959                 .append(nextHop).toString();
1960     }
1961
1962     protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
1963                                             final VrfEntry vrfEntry, String rd) {
1964         List<String> adjacencyList = new ArrayList<>();
1965         List<String> prefixIpList = new ArrayList<>();
1966         LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
1967                 remoteDpnId, vpnId, vrfEntry);
1968         try {
1969             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1970                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1971                 if (extra_route == null) {
1972                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1973                 } else {
1974                     prefixIpList = new ArrayList<>();
1975                     for (String extraRouteIp : extra_route.getNexthopIpList()) {
1976                         prefixIpList.add(extraRouteIp + "/32");
1977                     }
1978                 }
1979             } else {
1980                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1981             }
1982
1983             for (String prefixIp : prefixIpList) {
1984                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
1985                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
1986                     String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
1987                             prefixIp, nextHopIp);
1988                     if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
1989                         adjacencyList.add(adjacency);
1990                     }
1991                 }
1992             }
1993         } catch (NullPointerException e) {
1994             LOG.trace("", e);
1995         }
1996         return adjacencyList;
1997     }
1998
1999     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2000         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2001                 InstanceIdentifier.create(VpnInstanceOpData.class)
2002                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2003         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2004                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2005         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2006     }
2007
2008     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2009         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2010                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2011                 .append(FLOWID_PREFIX).toString();
2012     }
2013
2014     /*
2015      * Install flow entry in protocol table to forward mpls
2016      * coming through gre tunnel to LFIB table.
2017      */
2018     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2019         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2020         // Instruction to goto L3 InterfaceTable
2021         List<InstructionInfo> instructions = new ArrayList<>();
2022         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2023         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2024         matches.add(new MatchInfo(MatchFieldType.eth_type,
2025                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2026         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2027                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2028                         NwConstants.L3_LFIB_TABLE),
2029                 DEFAULT_FIB_FLOW_PRIORITY,
2030                 "Protocol Table For LFIB",
2031                 0, 0,
2032                 COOKIE_PROTOCOL_TABLE,
2033                 matches, instructions);
2034
2035         if (addOrRemove == NwConstants.ADD_FLOW) {
2036             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2037             mdsalManager.installFlow(flowEntityToLfib);
2038         } else {
2039             mdsalManager.removeFlow(flowEntityToLfib);
2040         }
2041     }
2042
2043     public List<String> printFibEntries() {
2044         List<String> result = new ArrayList<>();
2045         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2046         result.add("-------------------------------------------------------------------");
2047         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2048         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2049         if (fibEntries.isPresent()) {
2050             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2051             for (VrfTables vrfTable : vrfTables) {
2052                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2053                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2054                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2055                                 vrfTable.getRouteDistinguisher(),
2056                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2057                     }
2058                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2059                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2060                                 vrfTable.getRouteDistinguisher(),
2061                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2062                     }
2063                 }
2064             }
2065         }
2066         return result;
2067     }
2068
2069
2070     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2071         InstanceIdentifier<VrfEntry> vrfEntryId =
2072                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2073                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2074         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2075         if (vrfEntry.isPresent())  {
2076             return vrfEntry.get();
2077         }
2078         return null;
2079     }
2080
2081     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2082         InstanceIdentifier<VrfEntry> vrfEntryId =
2083                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2084                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2085         return vrfEntryId;
2086     }
2087
2088     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2089             long vpnId, int addOrRemove) {
2090         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2091         if (routerInt != null && vpnToDpnList != null) {
2092             String routerId = routerInt.getUuid();
2093             String macAddress = routerInt.getMacAddress();
2094             String ipValue = routerInt.getIpAddress();
2095             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2096                     routerId, ipValue, macAddress);
2097             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2098                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2099                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2100                             new MacAddress(macAddress), addOrRemove);
2101                 }
2102             }
2103             return true;
2104         }
2105         return false;
2106     }
2107
2108     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2109                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2110         String[] subSplit = routerInternalIp.split("/");
2111
2112         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2113         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2114                 dpnId, routerInternalIp, vpnId, subSplit[0]);
2115
2116         List<MatchInfo> matches = new ArrayList<>();
2117
2118         matches.add(new MatchInfo(MatchFieldType.ip_proto, new long[] { IPProtocols.ICMP.intValue() }));
2119         matches.add(new MatchInfo(MatchFieldType.metadata,
2120                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
2121         matches.add(new MatchInfo(MatchFieldType.icmp_v4, new long[] { (short) 8, (short) 0 }));
2122         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
2123         matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { subSplit[0], "32" }));
2124
2125         List<ActionInfo> actionsInfos = new ArrayList<>();
2126
2127         // Set Eth Src and Eth Dst
2128         actionsInfos.add(new ActionInfo(ActionType.move_src_dst_eth, new String[] {}));
2129         actionsInfos.add(new ActionInfo(ActionType.set_field_eth_src, new String[] { routerMac.getValue() }));
2130
2131         // Move Ip Src to Ip Dst
2132         actionsInfos.add(new ActionInfo(ActionType.move_src_dst_ip, new String[] {}));
2133         actionsInfos.add(new ActionInfo(ActionType.set_source_ip, new String[] { subSplit[0], "32" }));
2134
2135         // Set the ICMP type to 0 (echo reply)
2136         actionsInfos.add(new ActionInfo(ActionType.set_icmp_type, new String[] { "0" }));
2137
2138         actionsInfos.add(new ActionInfo(ActionType.nx_load_in_port, new BigInteger[]{ BigInteger.ZERO }));
2139
2140         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit,
2141                 new String[] { Short.toString(NwConstants.L3_FIB_TABLE) }));
2142
2143         List<InstructionInfo> instructions = new ArrayList<>();
2144
2145         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2146
2147         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY;
2148         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
2149
2150         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2151                 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2152
2153         if (addOrRemove == NwConstants.ADD_FLOW) {
2154             mdsalManager.installFlow(flowEntity);
2155         } else {
2156             mdsalManager.removeFlow(flowEntity);
2157         }
2158     }
2159 }