vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.CheckedFuture;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import java.math.BigInteger;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import java.util.concurrent.LinkedBlockingQueue;
  28 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  29 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  33 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  34 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  35 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  36 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  37 import org.opendaylight.genius.mdsalutil.ActionInfo;
  38 import org.opendaylight.genius.mdsalutil.ActionType;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  41 import org.opendaylight.genius.mdsalutil.InstructionType;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  44 import org.opendaylight.genius.mdsalutil.MatchInfo;
  45 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  46 import org.opendaylight.genius.mdsalutil.NwConstants;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.genius.utils.ServiceIndex;
  49 import org.opendaylight.genius.utils.batching.ActionableResource;
  50 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  51 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  52 import org.opendaylight.genius.utils.batching.ResourceHandler;
  53 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 111 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 112 import org.opendaylight.yangtools.yang.binding.DataObject;
 113 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 114 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 115 import org.opendaylight.yangtools.yang.common.RpcResult;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119 public class VrfEntryListener extends AbstractDataChangeListener<VrfEntry> implements AutoCloseable, ResourceHandler {
 120     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 121     private static final String FLOWID_PREFIX = "L3.";
 122     private ListenerRegistration<DataChangeListener> listenerRegistration;
 123     private final DataBroker dataBroker;
 124     private final IMdsalApiManager mdsalManager;
 125     private IVpnManager vpnmanager;
 126     private NexthopManager nextHopManager;
 127     private ItmRpcService itmManager;
 128     private OdlInterfaceRpcService interfaceManager;
 129     private IdManagerService idManager;
 130     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 131     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 132     private static final int LFIB_INTERVPN_PRIORITY = 1;
 133     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 134     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 135     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 136     private static final int PERIODICITY = 500;
 137     private static Integer batchSize;
 138     private static Integer batchInterval;
 139     private static final int BATCH_SIZE = 1000;
 140     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 141     private ResourceBatchingManager resourceBatchingManager;
 142
 143     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 144                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 145                             final IdManagerService idManager) {
 146         super(VrfEntry.class);
 147         this.dataBroker = dataBroker;
 148         this.mdsalManager = mdsalApiManager;
 149         this.nextHopManager = nexthopManager;
 150         this.interfaceManager = interfaceManager;
 151         this.idManager = idManager;
 152         batchSize = Integer.getInteger("batch.size");
 153         if (batchSize == null) {
 154             batchSize = BATCH_SIZE;
 155         }
 156         batchInterval = Integer.getInteger("batch.wait.time");
 157         if (batchInterval == null) {
 158             batchInterval = PERIODICITY;
 159         }
 160         resourceBatchingManager = ResourceBatchingManager.getInstance();
 161         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 162     }
 163
 164     public void start() {
 165         LOG.info("{} start", getClass().getSimpleName());
 166         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 167                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 168     }
 169
 170     private InstanceIdentifier<VrfEntry> getWildCardPath() {
 171         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 172     }
 173
 174     @Override
 175     public void close() throws Exception {
 176         if (listenerRegistration != null) {
 177             listenerRegistration.close();
 178             listenerRegistration = null;
 179         }
 180         LOG.info("{} close", getClass().getSimpleName());
 181     }
 182
 183     @Override
 184     public DataBroker getResourceBroker() {
 185         return dataBroker;
 186     }
 187
 188     @Override
 189     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 190         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 191         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 192         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 193                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 194         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 195             createFibEntries(identifier, vrfEntry);
 196         } else {
 197             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 198             actResource.setAction(ActionableResource.CREATE);
 199             actResource.setInstanceIdentifier(identifier);
 200             actResource.setInstance(vrfEntry);
 201             vrfEntryBufferQ.add(actResource);
 202             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 203         }
 204         LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 205                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 206     }
 207
 208     @Override
 209     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 210         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 211         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 212         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 213                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 214         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 215             deleteFibEntries(identifier, vrfEntry);
 216         } else {
 217             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 218             actResource.setAction(ActionableResource.DELETE);
 219             actResource.setInstanceIdentifier(identifier);
 220             actResource.setInstance(vrfEntry);
 221             vrfEntryBufferQ.add(actResource);
 222             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 223         }
 224         LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 225                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 226         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 227     }
 228
 229     @Override
 230     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 231         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 232         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 233         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 234                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 235         if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 236             createFibEntries(identifier, update);
 237         } else {
 238             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 239             actResource.setAction(ActionableResource.UPDATE);
 240             actResource.setInstanceIdentifier(identifier);
 241             actResource.setInstance(update);
 242             actResource.setOldInstance(original);
 243             vrfEntryBufferQ.add(actResource);
 244         }
 245         LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 246                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 247     }
 248
 249     @Override
 250     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 251         if (vrfEntry instanceof VrfEntry) {
 252             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 253         }
 254     }
 255
 256     @Override
 257     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 258         if (vrfEntry instanceof VrfEntry) {
 259             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 260         }
 261     }
 262
 263     @Override
 264     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 265                        Object update) {
 266         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 267             createFibEntries(tx, identifier, (VrfEntry)update);
 268         }
 269     }
 270
 271     @Override
 272     public int getBatchSize() {
 273         return batchSize;
 274     }
 275
 276     @Override
 277     public int getBatchInterval() {
 278         return batchInterval;
 279     }
 280
 281     @Override
 282     public LogicalDatastoreType getDatastoreType() {
 283         return LogicalDatastoreType.CONFIGURATION;
 284     }
 285
 286     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 287         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 288
 289         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 290         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 291         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 292
 293         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 294         final Long vpnId = vpnInstance.getVpnId();
 295         final String rd = vrfTableKey.getRouteDistinguisher();
 296         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 297         if (subnetRoute != null) {
 298             final long elanTag = subnetRoute.getElantag();
 299             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 300                     rd, vrfEntry.getDestPrefix(), elanTag);
 301             if (vpnToDpnList != null) {
 302                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 303                 dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 304                         new Callable<List<ListenableFuture<Void>>>() {
 305                             @Override
 306                             public List<ListenableFuture<Void>> call() throws Exception {
 307                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 308                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 309                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 310                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 311                                     }
 312                                 }
 313                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 314                                 futures.add(tx.submit());
 315                                 return futures;
 316                             }
 317                         });
 318             }
 319             return;
 320         }
 321
 322         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 323             // When it is a leaked route, the LFIB and FIB goes a bit different.
 324             installInterVpnRouteInLFib(rd, vrfEntry);
 325             return;
 326         }
 327
 328         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 329
 330         if (vpnToDpnList != null) {
 331             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 332             dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 333                     new Callable<List<ListenableFuture<Void>>>() {
 334                         @Override
 335                         public List<ListenableFuture<Void>> call() throws Exception {
 336                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 337                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 338                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 339                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 340                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 341                                     }
 342                                 }
 343                             }
 344                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 345                             futures.add(tx.submit());
 346                             return futures;
 347                         }
 348                     });
 349         }
 350
 351         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 352         if ( optVpnUuid.isPresent() ) {
 353             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, optVpnUuid.get());
 354             if ( interVpnLink.isPresent() ) {
 355                 String vpnUuid = optVpnUuid.get();
 356                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 357                 if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid, interVpnLink.get()) ) {
 358                     // This is an static route that points to the other endpoint of an InterVpnLink
 359                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 360                     installRouteInInterVpnLink(interVpnLink.get(), vpnUuid, vrfEntry, vpnId);
 361                 }
 362             }
 363         }
 364     }
 365
 366
 367     /*
 368       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 369       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 370       provided by ResourceBatchingManager
 371      */
 372     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 373         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 374
 375         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 376         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 377         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 378
 379         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 380         final String rd = vrfTableKey.getRouteDistinguisher();
 381         if (vpnToDpnList != null) {
 382             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 383                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 384                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 385                 }
 386             }
 387         }
 388     }
 389
 390     /*
 391      * Returns true if the specified nexthop is the other endpoint in an
 392      * InterVpnLink, regarding one of the VPN's point of view.
 393      */
 394     private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) {
 395         return
 396                 interVpnLink != null
 397                         && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
 398                         && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
 399                         || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
 400                         && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
 401     }
 402
 403
 404     // FIXME: Refactoring needed here.
 405     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 406     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 407                                    int addOrRemove) {
 408         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 409         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 410
 411         String rd = vrfTableKey.getRouteDistinguisher();
 412         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 413         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 414             if (vpnInstance == null) {
 415                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 416                 return;
 417             }
 418         } else {
 419             Preconditions.checkNotNull(vpnInstance,
 420                     "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 421         }
 422         String vpnUuid = vpnInstance.getVpnInstanceName();
 423         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 424                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 425
 426         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 427         // there an interVpnLink for the involved vpn in order to make learn the new route to
 428         // the other part of the inter-vpn-link.
 429
 430         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 431         Optional<InterVpnLink> interVpnLink =
 432                 (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 433                         : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 434         if ( !interVpnLink.isPresent() ) {
 435             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 436             return;
 437         }
 438
 439         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 440         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 441         boolean proceed = (addOrRemove == NwConstants.DEL_FLOW )
 442                 || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 443                 && interVpnLink.get().isBgpRoutesLeaking() );
 444
 445         if ( proceed ) {
 446             String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) )
 447                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 448                     : vpnUuid;
 449
 450             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 451             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 452
 453             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 454                     InstanceIdentifier.builder(FibEntries.class)
 455                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 456                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 457                             .build();
 458             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 459                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 460                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 461                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 462                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 463                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 464                         .setLabel(label)
 465                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 466                         .build();
 467                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 468             } else {
 469                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 470                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 471             }
 472         }
 473     }
 474
 475     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 476         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 477         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 478         prefixBuilder.setDpnId(lri.getDpnId());
 479         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 480         prefixBuilder.setIpAddress(lri.getPrefix());
 481         // Increment the refCount here
 482         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 483                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 484         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 485         if (!isPresentInList) {
 486             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 487             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 488             vpnInstanceNames.add(vpnInstanceName);
 489             builder.setVpnInstanceList(vpnInstanceNames);
 490             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 491         } else {
 492             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 493         }
 494         return prefixBuilder.build();
 495     }
 496
 497     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 498                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 499         Boolean wrTxPresent = true;
 500         if (tx == null) {
 501             wrTxPresent = false;
 502             tx = dataBroker.newWriteOnlyTransaction();
 503         }
 504         synchronized (vrfEntry.getLabel().toString().intern()) {
 505             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 506             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 507                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 508
 509                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 510                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 511                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 512                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 513                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 514                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 515                         }
 516                     }
 517                 }
 518                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 519                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 520             }
 521         }
 522         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 523         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 524         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 525         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 526         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 527
 528         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 529             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 530             // reinitialize instructions list for LFIB Table
 531             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 532
 533             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 534             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 535             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 536             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 537
 538             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 539         }
 540         if (!wrTxPresent ) {
 541             tx.submit();
 542         }
 543     }
 544
 545     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 546         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 547         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 548         // packet is commuted from Vpn2 to Vpn1.
 549         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 550         if ( !vpnNameOpc.isPresent() ) {
 551             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 552             return;
 553         }
 554
 555         String vpnName = vpnNameOpc.get();
 556         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 557         boolean interVpnLinkFound = false;
 558         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 559             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 560             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 561                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 562             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 563                 interVpnLinkFound = true;
 564
 565                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 566                 if ( !vpnLinkState.isPresent()
 567                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 568                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 569                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 570                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 571                     return;
 572                 }
 573
 574                 List<BigInteger> targetDpns =
 575                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
 576                                 : vpnLinkState.get().getSecondEndpointState().getDpId();
 577                 Long lportTag =
 578                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 579                                 : vpnLinkState.get().getFirstEndpointState().getLportTag();
 580
 581                 for ( BigInteger dpId : targetDpns ) {
 582                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 583
 584                     BigInteger[] metadata = new BigInteger[] {
 585                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 586                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 587                     };
 588                     List<InstructionInfo> instructions =
 589                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 590                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 591                                     new InstructionInfo(InstructionType.goto_table,
 592                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 593
 594                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 595                             NwConstants.ADD_FLOW, null);
 596                 }
 597
 598                 break;
 599             }
 600         }
 601
 602         if ( !interVpnLinkFound ) {
 603             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 604                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 605         }
 606     }
 607
 608
 609
 610     private void installRouteInInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 611                                             final VrfEntry vrfEntry, long vpnTag) {
 612         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 613         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 614                 && vrfEntry.getNextHopAddressList().size() == 1);
 615         String destination = vrfEntry.getDestPrefix();
 616         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 617
 618         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 619         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 620         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 621         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 622         if ( !interVpnLinkState.isPresent() ) {
 623             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 624             return;
 625         }
 626         if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 627             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 628                     destination, nextHop, interVpnLink.getName());
 629             return;
 630         }
 631
 632
 633         // Everything Ok
 634         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 635         List<BigInteger> targetDpns =
 636                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 637                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 638
 639         Long otherEndpointlportTag =
 640                 vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
 641                         : interVpnLinkState.get().getFirstEndpointState().getLportTag();
 642
 643         BigInteger[] metadata = new BigInteger[] {
 644                 MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag.intValue(),
 645                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 646                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 647         };
 648         int instIdx = 0;
 649         List<Instruction> instructions =
 650                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 651                         new InstructionInfo(InstructionType.goto_table,
 652                                 new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 653
 654         String values[] = destination.split("/");
 655         String destPrefixIpAddress = values[0];
 656         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 657
 658         List<MatchInfo> matches = new ArrayList<>();
 659         matches.add(new MatchInfo(MatchFieldType.metadata,
 660                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 661                         MetaDataUtil.METADATA_MASK_VRFID }));
 662         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 663
 664         if (prefixLength != 0) {
 665             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 666                     new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 667         }
 668
 669         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 670         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), destination, nextHop);
 671         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 672                 COOKIE_VM_FIB_TABLE, matches, instructions);
 673
 674         for ( BigInteger dpId : targetDpns ) {
 675             mdsalManager.installFlow(dpId, flowEntity);
 676         }
 677     }
 678
 679     private void removeRouteFromInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 680                                              final VrfEntry vrfEntry) {
 681
 682         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 683         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 684                 && vrfEntry.getNextHopAddressList().size() == 1);
 685
 686         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 687         if ( !interVpnLinkState.isPresent() ) {
 688             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 689             return;
 690         }
 691
 692         // Everything Ok
 693         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 694         List<BigInteger> targetDpns =
 695                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 696                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 697
 698         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 699         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 700         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
 701         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
 702                 .setFlowName(flowRef).build();
 703
 704         for ( BigInteger dpId : targetDpns ) {
 705             mdsalManager.removeFlow(dpId, flow);
 706         }
 707
 708     }
 709
 710     private boolean isVpnFirstEndPoint(InterVpnLink interVpnLink, String vpnName) {
 711         return interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 712     }
 713
 714     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 715                                                      InstanceIdentifier<T> path) {
 716
 717         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 718
 719         Optional<T> result = Optional.absent();
 720         try {
 721             result = tx.read(datastoreType, path).get();
 722         } catch (Exception e) {
 723             throw new RuntimeException(e);
 724         }
 725
 726         return result;
 727     }
 728
 729     private void makeSubnetRouteTableMissFlow(BigInteger dpnId, int addOrRemove) {
 730         final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 731         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 732         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 733         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[]{}));
 734         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 735         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 736         String flowRef = getTableMissFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, NwConstants.TABLE_MISS_FLOW);
 737         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef,
 738                 NwConstants.TABLE_MISS_PRIORITY, "Subnet Route Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
 739
 740         if (addOrRemove == NwConstants.ADD_FLOW) {
 741             mdsalManager.installFlow(flowEntity);
 742         } else {
 743             mdsalManager.removeFlow(flowEntity);
 744         }
 745     }
 746
 747     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 748         List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 749         Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 750         String localNextHopIP = vrfEntry.getDestPrefix();
 751
 752         if (localNextHopInfo == null) {
 753             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 754             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 755             if (extraRoute != null) {
 756                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 757                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 758                     if (nextHopIp != null) {
 759                         localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
 760                         localNextHopIP = nextHopIp + "/32";
 761                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 762                         returnLocalDpnId.add(dpnId);
 763                     }
 764                 }
 765             }
 766             if (localNextHopInfo == null) {
 767             /* imported routes case */
 768                 synchronized (vrfEntry.getLabel().toString().intern()) {
 769                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 770                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 771                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 772                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 773                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 774                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 775                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 776                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 777                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 778                                     localNextHopIP = lri.getPrefix();
 779                                 } else {
 780                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 781                                     localNextHopIP = lri.getPrefix();
 782                                 }
 783                             }
 784                             if (localNextHopInfo != null) {
 785                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 786                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 787                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 788                                 returnLocalDpnId.add(dpnId);
 789                             }
 790                         }
 791                     }
 792                 }
 793             }
 794         } else {
 795             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 796             returnLocalDpnId.add(dpnId);
 797         }
 798
 799         return returnLocalDpnId;
 800     }
 801
 802     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 803                                                 final VrfEntry vrfEntry, Long parentVpnId){
 804         if (localNextHopInfo != null) {
 805             final BigInteger dpnId = localNextHopInfo.getDpnId();
 806             if (!isVpnPresentInDpn(rd, dpnId)) {
 807                 return BigInteger.ZERO;
 808             }
 809
 810             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
 811
 812             List<ActionInfo> actionsInfos =
 813                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 814             final List<InstructionInfo> instructions =
 815                     Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 816             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 817                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 818             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 819             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 820                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 821                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 822             } else {
 823                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 824             }
 825             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 826             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 827                     new Callable<List<ListenableFuture<Void>>>() {
 828                         @Override
 829                         public List<ListenableFuture<Void>> call() throws Exception {
 830                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 831                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 832                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 833                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 834                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 835                             }
 836                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 837                             futures.add(tx.submit());
 838                             return futures;
 839                         }
 840                     });
 841             return dpnId;
 842         }
 843         return BigInteger.ZERO;
 844     }
 845
 846     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 847         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 848         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 849         if (dpnInVpn.isPresent()) {
 850             return true;
 851         }
 852         return false;
 853     }
 854
 855     private LabelRouteInfo getLabelRouteInfo(Long label) {
 856         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 857                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 858         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 859         if (opResult.isPresent()) {
 860             return opResult.get();
 861         }
 862         return null;
 863     }
 864
 865     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 866         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 867         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 868                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 869         if (lri == null) {
 870             return true;
 871         }
 872         List<String> vpnInstancesList = lri.getVpnInstanceList();
 873         if (vpnInstancesList.contains(vpnInstanceName)) {
 874             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 875             vpnInstancesList.remove(vpnInstanceName);
 876         }
 877         if (vpnInstancesList.size() == 0) {
 878             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 879             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 880             return true;
 881         } else {
 882             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 883             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 884             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 885         }
 886         return false;
 887     }
 888
 889     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 890                                       WriteTransaction tx) {
 891         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 892         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 893
 894
 895         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 896
 897         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 898                 dpId, label, groupId);
 899     }
 900
 901     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 902                                                  WriteTransaction tx) {
 903         List<MatchInfo> mkMatches = new ArrayList<>();
 904
 905         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 906
 907         // Matching metadata
 908         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 909         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 910
 911         List<InstructionInfo> mkInstructions = new ArrayList<>();
 912         mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 913
 914         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 915                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 916                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 917
 918         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 919
 920         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 921
 922         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 923         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 924                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 925                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 926         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 927     }
 928
 929     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 930         FlowEntity flowEntity;
 931         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 932         List<MatchInfo> mkMatches = new ArrayList<>();
 933         // Matching metadata
 934         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 935         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 936                 NwConstants.INTERNAL_TUNNEL_TABLE,
 937                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 938                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 939                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 940         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 941         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 942         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 943                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 944                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 945
 946         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 947         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 948     }
 949
 950     /**
 951      * Delete local FIB entry
 952      * @param vpnId
 953      * @param rd
 954      * @param vrfEntry
 955      * @return
 956      */
 957     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 958         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 959         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 960         String localNextHopIP = vrfEntry.getDestPrefix();
 961
 962         if (localNextHopInfo == null) {
 963             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 964             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 965             if (extra_route != null) {
 966                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 967                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 968                     if (nextHopIp != null) {
 969                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 970                         localNextHopIP = nextHopIp + "/32";
 971                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 972                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 973                         if (!dpnId.equals(BigInteger.ZERO)) {
 974                             returnLocalDpnId.add(dpnId);
 975                         }
 976                     }
 977                 }
 978             }
 979
 980             if (localNextHopInfo == null) {
 981               /* Imported VRF entry */
 982                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 983                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 984                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 985                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
 986                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
 987                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
 988                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 989                     if (!dpnId.equals(BigInteger.ZERO)) {
 990                         returnLocalDpnId.add(dpnId);
 991                     }
 992                 }
 993             }
 994
 995
 996         } else {
 997             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 998                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 999             if (!dpnId.equals(BigInteger.ZERO)) {
1000                 returnLocalDpnId.add(dpnId);
1001             }
1002         }
1003
1004         return returnLocalDpnId;
1005     }
1006
1007     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1008                                                 final Long vpnId, final String rd,
1009                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1010         if (localNextHopInfo != null) {
1011             final BigInteger dpnId = localNextHopInfo.getDpnId();;
1012             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1013             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
1014                     new Callable<List<ListenableFuture<Void>>>() {
1015                         @Override
1016                         public List<ListenableFuture<Void>> call() throws Exception {
1017                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1018                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1019                                     NwConstants.DEL_FLOW, tx);
1020                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1021                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1022                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1023                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1024                             }
1025                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1026                             futures.add(tx.submit());
1027                             return futures;
1028                         }
1029                     });
1030             //TODO: verify below adjacency call need to be optimized (?)
1031             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
1032             return dpnId;
1033         }
1034         return BigInteger.ZERO;
1035     }
1036
1037
1038     private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
1039         return InstanceIdentifier.builder(PrefixToInterface.class)
1040                 .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
1041     }
1042
1043     private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
1044         Optional<Prefixes> localNextHopInfoData =
1045                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
1046         return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
1047     }
1048
1049     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1050         return InstanceIdentifier.builder(VpnToExtraroute.class)
1051                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1052                         new ExtrarouteKey(ipPrefix)).build();
1053     }
1054
1055     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1056         Optional<Extraroute> extraRouteInfo =
1057                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1058         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1059
1060     }
1061
1062     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1063         try {
1064             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1065                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1066             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1067             if(!rpcResult.isSuccessful()) {
1068                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1069             } else {
1070                 return rpcResult.getResult().getTunnelType();
1071             }
1072
1073         } catch (InterruptedException | ExecutionException e) {
1074             LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
1075         }
1076
1077         return null;
1078
1079     }
1080     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1081                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1082         Boolean wrTxPresent = true;
1083         if (tx == null) {
1084             wrTxPresent = false;
1085             tx = dataBroker.newWriteOnlyTransaction();
1086         }
1087         String rd = vrfTableKey.getRouteDistinguisher();
1088         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1089                 vrfEntry.getDestPrefix(), rd, tx);
1090         /********************************************/
1091         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1092
1093         if (tunnelInterfaceList.isEmpty()) {
1094             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1095                     vrfEntry.getNextHopAddressList(), rd);
1096             LOG.warn("Failed to add Route: {} in vpn: {}",
1097                     vrfEntry.getDestPrefix(), rd);
1098             return;
1099         }
1100
1101         for (String tunnelInterface : tunnelInterfaceList) {
1102             List<InstructionInfo> instructions = new ArrayList<>();
1103             List<ActionInfo> actionInfos = new ArrayList<>();
1104             Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1105             if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1106                 LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1107                 actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
1108                 actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
1109             } else {
1110                 int label = vrfEntry.getLabel().intValue();
1111                 BigInteger tunnelId;
1112                 // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1113                 if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1114                     tunnelId = BigInteger.valueOf(label);
1115                 } else {
1116                     tunnelId = BigInteger.valueOf(label);
1117                 }
1118
1119                 LOG.debug("adding set tunnel id action for label {}", label);
1120                 actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
1121             }
1122             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
1123             if(egressActions.isEmpty()){
1124                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
1125                 return;
1126             }
1127             actionInfos.addAll(egressActions);
1128             instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
1129             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1130         }
1131         if(!wrTxPresent ){
1132             tx.submit();
1133         }
1134         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1135     }
1136
1137     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1138         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1139         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1140         if (dpnInVpn.isPresent()) {
1141             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1142             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1143
1144             if (vpnInterfaces.remove(currVpnInterface)) {
1145                 if (vpnInterfaces.isEmpty()) {
1146                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1147                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1148                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1149                 } else {
1150                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1151                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1152                             VpnInterfaces.class,
1153                             new VpnInterfacesKey(intfName)));
1154                 }
1155             }
1156         }
1157     }
1158
1159     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1160     /* Get interface info from prefix to interface mapping;
1161         Use the interface info to get the corresponding vpn interface op DS entry,
1162         remove the adjacency corresponding to this fib entry.
1163         If adjacency removed is the last adjacency, clean up the following:
1164          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1165          - prefix to interface entry
1166          - vpn interface op DS
1167      */
1168         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1169         Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1170         Extraroute extraRoute = null;
1171         if (prefixInfo == null) {
1172             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1173             if(extraRoute != null) {
1174                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1175                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1176
1177                     if (nextHopIp != null) {
1178                         prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
1179                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1180                     }
1181                 }
1182             }
1183             if (prefixInfo == null) {
1184                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1185                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1186                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1187                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1188                     prefixBuilder.setDpnId(lri.getDpnId());
1189                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1190                     prefixBuilder.setIpAddress(lri.getPrefix());
1191                     prefixInfo = prefixBuilder.build();
1192                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1193                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1194                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1195                 }
1196             }
1197         } else {
1198             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1199         }
1200     }
1201
1202     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1203                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1204
1205         if (prefixInfo == null) {
1206             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1207             return; //Don't have any info for this prefix (shouldn't happen); need to return
1208         }
1209
1210         String ifName = prefixInfo.getVpnInterfaceName();
1211         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1212         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1213                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1214     }
1215
1216     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1217         Prefixes prefixInfo;
1218         Long vpnId;
1219         String rd;
1220         VrfEntry vrfEntry;
1221         Extraroute extraRoute;
1222
1223         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1224                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1225             this.prefixInfo = prefixInfo;
1226             this.vpnId = vpnId;
1227             this.rd= rd;
1228             this.vrfEntry= vrfEntry;
1229             this.extraRoute = extraRoute;
1230         }
1231
1232         @Override
1233         public List<ListenableFuture<Void>> call() throws Exception {
1234             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1235             // to call the respective helpers.
1236             String ifName = prefixInfo.getVpnInterfaceName();
1237             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1238                     FibUtil.getVpnInterfaceIdentifier(ifName));
1239             if (optvpnInterface.isPresent()) {
1240                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1241                 if (vpnId != associatedVpnId) {
1242                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1243                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1244                     LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
1245                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1246                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1247                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1248                     return null;
1249                 } else {
1250                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1251                             vrfEntry.getDestPrefix(), associatedVpnId);
1252                 }
1253             }
1254             if (extraRoute != null) {
1255                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1256                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1257             }
1258             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1259                     FibUtil.getAdjListPath(ifName));
1260             int numAdj = 0;
1261             if (optAdjacencies.isPresent()) {
1262                 numAdj = optAdjacencies.get().getAdjacency().size();
1263             }
1264             //remove adjacency corr to prefix
1265             if (numAdj > 1) {
1266                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1267                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1268                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1269             }
1270             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1271                 //clean up the vpn interface from DpnToVpn list
1272                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1273                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1274                         FibUtil.getVpnInterfaceIdentifier(ifName));
1275             }
1276
1277             synchronized (vrfEntry.getLabel().toString().intern()) {
1278                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1279                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1280                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1281                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1282                     String vpnInstanceName = "";
1283                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1284                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1285                     }
1286                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1287                     if (lriRemoved) {
1288                         String parentRd = lri.getParentVpnRd();
1289                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1290                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1291                     }
1292                 } else {
1293                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1294                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1295                 }
1296             }
1297             return null;
1298         }
1299     }
1300
1301     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1302         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1303         final String rd  = vrfTableKey.getRouteDistinguisher();
1304         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1305         if (vpnInstance == null) {
1306             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1307             return;
1308         }
1309         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1310         long elanTag = 0L;
1311         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1312         if (subnetRoute != null) {
1313             elanTag = subnetRoute.getElantag();
1314             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1315                     rd, vrfEntry.getDestPrefix(), elanTag);
1316             if (vpnToDpnList != null) {
1317                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1318                 dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1319                         new Callable<List<ListenableFuture<Void>>>() {
1320                             @Override
1321                             public List<ListenableFuture<Void>> call() throws Exception {
1322                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1323
1324                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1325
1326                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1327                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1328                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1329                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1330                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1331                                     }
1332                                 }
1333                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1334                                 futures.add(tx.submit());
1335                                 return futures;
1336                             }
1337                         });
1338             }
1339             synchronized (vrfEntry.getLabel().toString().intern()) {
1340                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1341                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1342                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1343                     String vpnInstanceName = "";
1344                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1345                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1346                     }
1347                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1348                     if (lriRemoved) {
1349                         String parentRd = lri.getParentVpnRd();
1350                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1351                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1352                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1353                                 vrfEntry.getDestPrefix());
1354                     }
1355                 } else {
1356                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1357                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1358                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1359                             vrfEntry.getDestPrefix());
1360                 }
1361             }
1362             return;
1363         }
1364
1365         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1366                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1367         if (vpnToDpnList != null) {
1368             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1369             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1370                     new Callable<List<ListenableFuture<Void>>>() {
1371                         @Override
1372                         public List<ListenableFuture<Void>> call() throws Exception {
1373                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1374
1375                             if (localDpnIdList.size() <= 0) {
1376                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1377                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1378                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1379                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1380                                         }
1381                                     } else {
1382                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1383                                     }
1384                                 }
1385                             } else {
1386                                 for (BigInteger localDpnId : localDpnIdList) {
1387                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1388                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1389                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1390                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1391                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1392                                                 }
1393                                             } else {
1394                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1395                                             }
1396                                         }
1397                                     }
1398                                 }
1399                             }
1400                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1401                             futures.add(tx.submit());
1402                             return futures;
1403                         }
1404                     });
1405         }
1406
1407         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1408         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1409         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1410
1411         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1412         // of the interVpnLink.
1413         Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
1414         if ( vpnUuid.isPresent() ) {
1415             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, vpnUuid.get());
1416             String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
1417
1418             if ( interVpnLink.isPresent()
1419                     && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
1420                     && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
1421                     || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
1422                     && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
1423                 // This is route that points to the other endpoint of an InterVpnLink
1424                 // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1425                 removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
1426             }
1427         }
1428
1429     }
1430
1431     /*
1432       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1433       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1434       provided by ResourceBatchingManager
1435      */
1436     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1437         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1438
1439         final String rd  = vrfTableKey.getRouteDistinguisher();
1440         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1441         if (vpnInstance == null) {
1442             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1443             return;
1444         }
1445         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1446         if (vpnToDpnList != null) {
1447             for (VpnToDpnList curDpn : vpnToDpnList) {
1448                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1449                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1450                 }
1451             }
1452         }
1453     }
1454
1455     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1456                                   final long vpnId, final VrfTablesKey vrfTableKey,
1457                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1458
1459         Boolean wrTxPresent = true;
1460         if (tx == null) {
1461             wrTxPresent = false;
1462             tx = dataBroker.newWriteOnlyTransaction();
1463         }
1464
1465         LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1466         String rd = vrfTableKey.getRouteDistinguisher();
1467
1468         if(localDpnId != null) {
1469             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1470             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1471             return;
1472         }
1473
1474         // below two reads are kept as is, until best way is found to identify dpnID
1475         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1476         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1477
1478         if (localNextHopInfo == null && extraRoute != null) {
1479             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1480             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1481                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1482                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1483             }
1484         } else {
1485             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1486         }
1487         if(!wrTxPresent ){
1488             tx.submit();
1489         }
1490     }
1491
1492     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1493                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1494         boolean isRemoteRoute = true;
1495         if (localNextHopInfo != null) {
1496             isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId()));
1497         }
1498         if (isRemoteRoute) {
1499             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1500             return true;
1501         } else {
1502             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1503             return false;
1504         }
1505     }
1506
1507     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1508         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1509         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1510     }
1511
1512     private long get
1513             (byte[] rawIpAddress) {
1514         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1515                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1516     }
1517
1518     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1519                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1520         Boolean wrTxPresent = true;
1521         if (tx == null) {
1522             wrTxPresent = false;
1523             tx = dataBroker.newWriteOnlyTransaction();
1524         }
1525
1526         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1527         String values[] = vrfEntry.getDestPrefix().split("/");
1528         String ipAddress = values[0];
1529         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1530         if (addOrRemove == NwConstants.ADD_FLOW) {
1531             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1532         } else {
1533             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1534         }
1535         InetAddress destPrefix;
1536         try {
1537             destPrefix = InetAddress.getByName(ipAddress);
1538         } catch (UnknownHostException e) {
1539             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1540             return;
1541         }
1542
1543         List<MatchInfo> matches = new ArrayList<>();
1544
1545         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1546                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1547
1548         matches.add(new MatchInfo(MatchFieldType.eth_type,
1549                 new long[] { NwConstants.ETHTYPE_IPV4 }));
1550
1551         if(prefixLength != 0) {
1552             matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1553                     destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1554         }
1555         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1556         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1557         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1558                 COOKIE_VM_FIB_TABLE, matches, instructions);
1559
1560         Flow flow = flowEntity.getFlowBuilder().build();
1561         String flowId = flowEntity.getFlowId();
1562         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1563         Node nodeDpn = buildDpnNode(dpId);
1564
1565         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1566                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1567                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1568         if (addOrRemove == NwConstants.ADD_FLOW) {
1569             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1570         } else {
1571             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1572         }
1573
1574         if(!wrTxPresent ){
1575             tx.submit();
1576         }
1577     }
1578
1579     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1580     private Node buildDpnNode(BigInteger dpnId) {
1581         NodeId nodeId = new NodeId("openflow:" + dpnId);
1582         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1583
1584         return nodeDpn;
1585     }
1586
1587     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1588                                     int addOrRemove, WriteTransaction tx) {
1589         Boolean wrTxPresent = true;
1590         if (tx == null) {
1591             wrTxPresent = false;
1592             tx = dataBroker.newWriteOnlyTransaction();
1593         }
1594
1595         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1596         matches.add(new MatchInfo(MatchFieldType.eth_type,
1597                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1598         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1599
1600         // Install the flow entry in L3_LFIB_TABLE
1601         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1602
1603         FlowEntity flowEntity;
1604         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1605                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1606         Flow flow = flowEntity.getFlowBuilder().build();
1607         String flowId = flowEntity.getFlowId();
1608         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1609         Node nodeDpn = buildDpnNode(dpId);
1610         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1611                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1612                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1613
1614         if (addOrRemove == NwConstants.ADD_FLOW) {
1615             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1616         } else {
1617             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1618         }
1619         if(!wrTxPresent ){
1620             tx.submit();
1621         }
1622         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
1623                 dpId, label, instructions );
1624     }
1625
1626     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
1627         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1628         try {
1629             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
1630         } catch (NullPointerException e) {
1631             LOG.trace("", e);
1632         }
1633     }
1634
1635     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1636                                     final FutureCallback<List<Void>> callback) {
1637         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1638         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1639         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1640         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1641         if (vrfTable.isPresent()) {
1642             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1643             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1644                     new Callable<List<ListenableFuture<Void>>>() {
1645                         @Override
1646                         public List<ListenableFuture<Void>> call() throws Exception {
1647                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1648                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1649                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1650                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1651
1652                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1653                                     if (subnetRoute != null) {
1654                                         long elanTag = subnetRoute.getElantag();
1655                                         installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1656                                         continue;
1657                                     }
1658                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1659                                         //Handle local flow creation for imports
1660                                         LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1661                                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()
1662                                         ) && vrfEntry.getNextHopAddressList()
1663                                                 .contains(lri.getNextHopIpList().get(0))) {
1664                                             if (lri.getDpnId().equals(dpnId)) {
1665                                                 createLocalFibEntry(vpnId, rd, vrfEntry);
1666                                                 continue;
1667                                             }
1668                                         }
1669                                     }
1670                                     // Passing null as we don't know the dpn
1671                                     // to which prefix is attached at this point
1672                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1673                                 }
1674                                 //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1675                                 futures.add(tx.submit());
1676                                 if (callback != null) {
1677                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1678                                     Futures.addCallback(listenableFuture, callback);
1679                                 }
1680                             }
1681                             return futures;
1682                         }
1683                     });
1684         }
1685     }
1686
1687     public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1688                                  final String localNextHopIp, final String remoteNextHopIp) {
1689         LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1690                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1691         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1692         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1693         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1694         if (vrfTable.isPresent()) {
1695             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1696             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1697                     new Callable<List<ListenableFuture<Void>>>() {
1698                         @Override
1699                         public List<ListenableFuture<Void>> call() throws Exception {
1700                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1701                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1702                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1703                                 LOG.trace("populate FIB starts on Dpn " + dpnId
1704                                         + "rd  " + rd.toString()
1705                                         + "localNextHopIp " + localNextHopIp
1706                                         + "remoteNextHopIp" + remoteNextHopIp
1707                                         + "vpnId " + vpnId );
1708                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1709                                     LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1710
1711                                     if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1712                                         if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1713                                             LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1714                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1715                                         }
1716                                     } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
1717                                         BigInteger dpnIdForPrefix = null;
1718                                         String destPfx = vrfEntry.getDestPrefix();
1719                                         if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
1720                                             Optional<Extraroute> extraRouteInfo =
1721                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1722                                                             getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1723                                             if (extraRouteInfo.isPresent()) {
1724                                                 continue;
1725                                             }
1726                                             dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1727                                         } else {
1728                                             // Subnet Route handling
1729                                             Optional<Prefixes> localNextHopInfoData =
1730                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1731                                                             FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
1732                                             if (localNextHopInfoData.isPresent()) {
1733                                                 Prefixes prefixes = localNextHopInfoData.get();
1734                                                 dpnIdForPrefix = prefixes.getDpnId();
1735                                             }
1736                                         }
1737                                         if (dpnIdForPrefix == null) {
1738                                             LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1739                                                     vrfEntry.getDestPrefix());
1740                                             continue;
1741                                         }
1742                                         int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1743                                         if (sameDpnId != 0) {
1744                                             LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1745                                                     dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1746                                             continue;
1747                                         }
1748                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1749                                         List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
1750                                         newNextHopAddrList.add(localNextHopIp);
1751                                         VrfEntry newVrfEntry =
1752                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
1753                                         // Just update the VrfEntry
1754                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1755                                                 vrfEntryId, newVrfEntry);
1756                                         // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
1757                                         //       vrfEntryId, newVrfEntry);
1758                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1759                                         LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1760                                     }
1761                                 }
1762                                 futures.add(writeTransaction.submit());
1763                                 LOG.trace("populate FIB ends on Dpn " + dpnId
1764                                         + "rd  " + rd.toString()
1765                                         + "localNextHopIp " + localNextHopIp
1766                                         + "remoteNextHopIp" + remoteNextHopIp
1767                                         + "vpnId " + vpnId);
1768                             }
1769                             return futures;
1770                         }
1771                     });
1772         }
1773     }
1774
1775     public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
1776                                   final long vpnId, final String  rd, final String destPrefix ,
1777                                   final String localNextHopIP, final String remoteNextHopIp) {
1778
1779         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1780         dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1781                         + "local dpid" + localDpnId
1782                         + "remote dpid" + remoteDpnId
1783                         + "vpnId" + vpnId
1784                         + "localNHIp" + localNextHopIP
1785                         + "remoteNHIp" + remoteNextHopIp,
1786                 new Callable<List<ListenableFuture<Void>>>() {
1787                     @Override
1788                     public List<ListenableFuture<Void>> call() throws Exception {
1789                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1790                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1791                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1792                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1793                         if (vrfEntry == null)
1794                             return futures;
1795                         LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1796                                         "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1797                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1798                         if (action == true) {
1799                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1800                             LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1801                             createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1802                         } else {
1803                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1804                             LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1805                             deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1806                         }
1807                         futures.add(writeTransaction.submit());
1808                         return futures;
1809                     }
1810                 });
1811     }
1812
1813     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1814                                  final FutureCallback<List<Void>> callback) {
1815         LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1816         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1817         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1818         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1819         if (vrfTable.isPresent()) {
1820             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1821             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1822                     new Callable<List<ListenableFuture<Void>>>() {
1823                         @Override
1824                         public List<ListenableFuture<Void>> call() throws Exception {
1825                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1826                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1827                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1828                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1829                                     /* Handle subnet routes here */
1830                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1831                                     if (subnetRoute != null) {
1832                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1833                                                 dpnId, rd);
1834                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1835                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1836                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1837                                                 vrfEntry.getDestPrefix());
1838                                         continue;
1839                                     }
1840                                     // Passing null as we don't know the dpn
1841                                     // to which prefix is attached at this point
1842                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1843                                 }
1844                                 futures.add(tx.submit());
1845                                 if (callback != null) {
1846                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1847                                     Futures.addCallback(listenableFuture, callback);
1848                                 }
1849                             }
1850                             return futures;
1851                         }
1852
1853                     });
1854         }
1855     }
1856
1857     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1858                                  final String localNextHopIp, final String remoteNextHopIp,
1859                                  final FutureCallback<List<Void>> callback) {
1860         LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1861                         " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1862                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1863         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1864         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1865         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1866         if (vrfTable.isPresent()) {
1867             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1868             dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
1869                     new Callable<List<ListenableFuture<Void>>>() {
1870                         @Override
1871                         public List<ListenableFuture<Void>> call() throws Exception {
1872                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1873                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1874                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1875                                 LOG.trace("cleanup FIB starts on Dpn " + dpnId
1876                                         + "rd  " + rd.toString()
1877                                         + "localNextHopIp " + localNextHopIp
1878                                         + "remoteNextHopIp" + remoteNextHopIp
1879                                         + "vpnId " + vpnId);
1880
1881                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1882                                     LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1883                                     if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1884                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1885                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1886                                     }
1887
1888                                     if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1889                                         LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1890                                                 vrfEntry.getDestPrefix(), dpnId);
1891                                         String destPfx = vrfEntry.getDestPrefix();
1892                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1893                                         List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1894                                         newList.remove(localNextHopIp);
1895                                         VrfEntry newVrfEntry =
1896                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1897                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1898                                                 vrfEntryId, newVrfEntry);
1899                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1900                                         LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1901                                     }
1902                                 }
1903                                 futures.add(writeTransaction.submit());
1904                                 if (callback != null) {
1905                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1906                                     Futures.addCallback(listenableFuture, callback);
1907                                 }
1908                                 LOG.trace("cleanup FIB ends on Dpn " + dpnId
1909                                         + "rd  " + rd.toString()
1910                                         + "localNextHopIp " + localNextHopIp
1911                                         + "remoteNextHopIp" + remoteNextHopIp
1912                                         + "vpnId " + vpnId);
1913                             }
1914                             return futures;
1915                         }
1916                     });
1917
1918         }
1919     }
1920
1921     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1922         InstanceIdentifierBuilder<VrfTables> idBuilder =
1923                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1924         InstanceIdentifier<VrfTables> id = idBuilder.build();
1925         return id;
1926     }
1927
1928     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1929         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1930                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1931                 .append(priority).toString();
1932     }
1933
1934     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1935         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1936                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1937                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1938                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1939                 .append(destPrefix.getHostAddress()).toString();
1940     }
1941
1942     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1943         return new StringBuilder(64).append(FLOWID_PREFIX)
1944                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1945                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1946                 .append(nextHop).toString();
1947     }
1948
1949     protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
1950                                             final VrfEntry vrfEntry, String rd) {
1951         List<String> adjacencyList = new ArrayList<>();
1952         List<String> prefixIpList = new ArrayList<>();
1953         LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
1954                 remoteDpnId, vpnId, vrfEntry);
1955         try {
1956             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1957                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1958                 if (extra_route == null) {
1959                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1960                 } else {
1961                     prefixIpList = new ArrayList<>();
1962                     for (String extraRouteIp : extra_route.getNexthopIpList()) {
1963                         prefixIpList.add(extraRouteIp + "/32");
1964                     }
1965                 }
1966             } else {
1967                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1968             }
1969
1970             for (String prefixIp : prefixIpList) {
1971                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
1972                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
1973                     String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
1974                             prefixIp, nextHopIp);
1975                     if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
1976                         adjacencyList.add(adjacency);
1977                     }
1978                 }
1979             }
1980         } catch (NullPointerException e) {
1981             LOG.trace("", e);
1982         }
1983         return adjacencyList;
1984     }
1985
1986     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
1987         InstanceIdentifier<VpnInstanceOpDataEntry> id =
1988                 InstanceIdentifier.create(VpnInstanceOpData.class)
1989                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
1990         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
1991                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1992         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
1993     }
1994
1995     public void processNodeAdd(BigInteger dpnId) {
1996         LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId);
1997         makeTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1998         makeL3IntfTblMissFlow(dpnId, NwConstants.ADD_FLOW);
1999         makeSubnetRouteTableMissFlow(dpnId, NwConstants.ADD_FLOW);
2000     }
2001
2002     private void makeTableMissFlow(BigInteger dpnId, int addOrRemove) {
2003         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2004         // Instruction to goto L3 InterfaceTable
2005         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2006         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE }));
2007         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2008         FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_LFIB_TABLE,
2009                 getTableMissFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW),
2010                 NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
2011
2012         FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE,
2013                 getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE,
2014                         NwConstants.TABLE_MISS_FLOW),
2015                 NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow",
2016                 0, 0, COOKIE_VM_FIB_TABLE,
2017                 matches, instructions);
2018
2019         if (addOrRemove == NwConstants.ADD_FLOW) {
2020             LOG.debug("Invoking MDSAL to install Table Miss Entries");
2021             mdsalManager.installFlow(flowEntityLfib);
2022             mdsalManager.installFlow(flowEntityFib);
2023         } else {
2024             mdsalManager.removeFlow(flowEntityLfib);
2025             mdsalManager.removeFlow(flowEntityFib);
2026
2027         }
2028     }
2029
2030     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2031         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2032                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2033                 .append(FLOWID_PREFIX).toString();
2034     }
2035
2036     /*
2037      * Install flow entry in protocol table to forward mpls
2038      * coming through gre tunnel to LFIB table.
2039      */
2040     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2041         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2042         // Instruction to goto L3 InterfaceTable
2043         List<InstructionInfo> instructions = new ArrayList<>();
2044         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2045         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2046         matches.add(new MatchInfo(MatchFieldType.eth_type,
2047                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2048         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2049                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2050                         NwConstants.L3_LFIB_TABLE),
2051                 DEFAULT_FIB_FLOW_PRIORITY,
2052                 "Protocol Table For LFIB",
2053                 0, 0,
2054                 COOKIE_PROTOCOL_TABLE,
2055                 matches, instructions);
2056
2057         if (addOrRemove == NwConstants.ADD_FLOW) {
2058             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2059             mdsalManager.installFlow(flowEntityToLfib);
2060         } else {
2061             mdsalManager.removeFlow(flowEntityToLfib);
2062         }
2063     }
2064
2065     public List<String> printFibEntries() {
2066         List<String> result = new ArrayList<String>();
2067         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2068         result.add("-------------------------------------------------------------------");
2069         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2070         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2071         if (fibEntries.isPresent()) {
2072             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2073             for (VrfTables vrfTable : vrfTables) {
2074                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2075                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2076                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2077                                 vrfTable.getRouteDistinguisher(),
2078                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2079                     }
2080                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2081                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2082                                 vrfTable.getRouteDistinguisher(),
2083                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2084                     }
2085                 }
2086             }
2087         }
2088         return result;
2089     }
2090
2091     private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) {
2092         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2093         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2094         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2095         // Instruction to goto L3 InterfaceTable
2096
2097         List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
2098         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
2099                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
2100         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2101         //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
2102
2103         FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
2104                 getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
2105                 NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS,
2106                 matches, instructions);
2107         if (addOrRemove == NwConstants.ADD_FLOW) {
2108             LOG.debug("Invoking MDSAL to install L3 interface Table Miss Entries");
2109             mdsalManager.installFlow(flowEntityL3Intf);
2110         } else {
2111             mdsalManager.removeFlow(flowEntityL3Intf);
2112         }
2113     }
2114
2115     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2116         InstanceIdentifier<VrfEntry> vrfEntryId =
2117                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2118                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2119         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2120         if (vrfEntry.isPresent())  {
2121             return (vrfEntry.get());
2122         }
2123         return null;
2124     }
2125
2126     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2127         InstanceIdentifier<VrfEntry> vrfEntryId =
2128                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2129                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2130         return vrfEntryId;
2131     }
2132 }