vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.CheckedFuture;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import java.math.BigInteger;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import java.util.concurrent.LinkedBlockingQueue;
  28 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  29 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  33 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  34 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  35 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  36 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  37 import org.opendaylight.genius.mdsalutil.ActionInfo;
  38 import org.opendaylight.genius.mdsalutil.ActionType;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  41 import org.opendaylight.genius.mdsalutil.InstructionType;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  44 import org.opendaylight.genius.mdsalutil.MatchInfo;
  45 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  46 import org.opendaylight.genius.mdsalutil.NwConstants;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.genius.utils.ServiceIndex;
  49 import org.opendaylight.genius.utils.batching.ActionableResource;
  50 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  51 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  52 import org.opendaylight.genius.utils.batching.ResourceHandler;
  53 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 111 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 112 import org.opendaylight.yangtools.yang.binding.DataObject;
 113 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 114 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 115 import org.opendaylight.yangtools.yang.common.RpcResult;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119 public class VrfEntryListener extends AbstractDataChangeListener<VrfEntry> implements AutoCloseable, ResourceHandler {
 120     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 121     private static final String FLOWID_PREFIX = "L3.";
 122     private ListenerRegistration<DataChangeListener> listenerRegistration;
 123     private final DataBroker dataBroker;
 124     private final IMdsalApiManager mdsalManager;
 125     private IVpnManager vpnmanager;
 126     private NexthopManager nextHopManager;
 127     private ItmRpcService itmManager;
 128     private OdlInterfaceRpcService interfaceManager;
 129     private IdManagerService idManager;
 130     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 131     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 132     private static final int LFIB_INTERVPN_PRIORITY = 1;
 133     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 134     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 135     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 136     private static final int PERIODICITY = 500;
 137     private static Integer batchSize;
 138     private static Integer batchInterval;
 139     private static final int BATCH_SIZE = 1000;
 140     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 141     private ResourceBatchingManager resourceBatchingManager;
 142
 143     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 144                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 145                             final IdManagerService idManager) {
 146         super(VrfEntry.class);
 147         this.dataBroker = dataBroker;
 148         this.mdsalManager = mdsalApiManager;
 149         this.nextHopManager = nexthopManager;
 150         this.interfaceManager = interfaceManager;
 151         this.idManager = idManager;
 152         batchSize = Integer.getInteger("batch.size");
 153         if (batchSize == null) {
 154             batchSize = BATCH_SIZE;
 155         }
 156         batchInterval = Integer.getInteger("batch.wait.time");
 157         if (batchInterval == null) {
 158             batchInterval = PERIODICITY;
 159         }
 160         resourceBatchingManager = ResourceBatchingManager.getInstance();
 161         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 162     }
 163
 164     public void start() {
 165         LOG.info("{} start", getClass().getSimpleName());
 166         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 167                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 168     }
 169
 170     private InstanceIdentifier<VrfEntry> getWildCardPath() {
 171         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 172     }
 173
 174     @Override
 175     public void close() throws Exception {
 176         if (listenerRegistration != null) {
 177             listenerRegistration.close();
 178             listenerRegistration = null;
 179         }
 180         LOG.info("{} close", getClass().getSimpleName());
 181     }
 182
 183     @Override
 184     public DataBroker getResourceBroker() {
 185         return dataBroker;
 186     }
 187
 188     @Override
 189     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 190         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 191         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 192         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 193                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 194         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 195             createFibEntries(identifier, vrfEntry);
 196         } else {
 197             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 198             actResource.setAction(ActionableResource.CREATE);
 199             actResource.setInstanceIdentifier(identifier);
 200             actResource.setInstance(vrfEntry);
 201             vrfEntryBufferQ.add(actResource);
 202             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 203         }
 204         LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 205                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 206     }
 207
 208     @Override
 209     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 210         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 211         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 212         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 213                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 214         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 215             deleteFibEntries(identifier, vrfEntry);
 216         } else {
 217             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 218             actResource.setAction(ActionableResource.DELETE);
 219             actResource.setInstanceIdentifier(identifier);
 220             actResource.setInstance(vrfEntry);
 221             vrfEntryBufferQ.add(actResource);
 222             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 223         }
 224         LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 225                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 226     }
 227
 228     @Override
 229     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 230         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 231         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 232         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 233                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 234         if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 235             createFibEntries(identifier, update);
 236         } else {
 237             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 238             actResource.setAction(ActionableResource.UPDATE);
 239             actResource.setInstanceIdentifier(identifier);
 240             actResource.setInstance(update);
 241             actResource.setOldInstance(original);
 242             vrfEntryBufferQ.add(actResource);
 243         }
 244         LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 245                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 246     }
 247
 248     @Override
 249     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 250         if (vrfEntry instanceof VrfEntry) {
 251             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 252         }
 253     }
 254
 255     @Override
 256     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 257         if (vrfEntry instanceof VrfEntry) {
 258             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 259         }
 260     }
 261
 262     @Override
 263     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 264                        Object update) {
 265         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 266             createFibEntries(tx, identifier, (VrfEntry)update);
 267         }
 268     }
 269
 270     @Override
 271     public int getBatchSize() {
 272         return batchSize;
 273     }
 274
 275     @Override
 276     public int getBatchInterval() {
 277         return batchInterval;
 278     }
 279
 280     @Override
 281     public LogicalDatastoreType getDatastoreType() {
 282         return LogicalDatastoreType.CONFIGURATION;
 283     }
 284
 285     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 286         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 287
 288         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 289         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 290         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 291
 292         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 293         final Long vpnId = vpnInstance.getVpnId();
 294         final String rd = vrfTableKey.getRouteDistinguisher();
 295         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 296         if (subnetRoute != null) {
 297             final long elanTag = subnetRoute.getElantag();
 298             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 299                     rd, vrfEntry.getDestPrefix(), elanTag);
 300             if (vpnToDpnList != null) {
 301                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 302                 dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 303                         new Callable<List<ListenableFuture<Void>>>() {
 304                             @Override
 305                             public List<ListenableFuture<Void>> call() throws Exception {
 306                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 307                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 308                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 309                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 310                                     }
 311                                 }
 312                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 313                                 futures.add(tx.submit());
 314                                 return futures;
 315                             }
 316                         });
 317             }
 318             return;
 319         }
 320
 321         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 322             // When it is a leaked route, the LFIB and FIB goes a bit different.
 323             installInterVpnRouteInLFib(rd, vrfEntry);
 324             return;
 325         }
 326
 327         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 328
 329         if (vpnToDpnList != null) {
 330             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 331             dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 332                     new Callable<List<ListenableFuture<Void>>>() {
 333                         @Override
 334                         public List<ListenableFuture<Void>> call() throws Exception {
 335                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 336                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 337                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 338                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 339                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 340                                     }
 341                                 }
 342                             }
 343                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 344                             futures.add(tx.submit());
 345                             return futures;
 346                         }
 347                     });
 348         }
 349
 350         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 351         if ( optVpnUuid.isPresent() ) {
 352             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, optVpnUuid.get());
 353             if ( interVpnLink.isPresent() ) {
 354                 String vpnUuid = optVpnUuid.get();
 355                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 356                 if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid, interVpnLink.get()) ) {
 357                     // This is an static route that points to the other endpoint of an InterVpnLink
 358                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 359                     installRouteInInterVpnLink(interVpnLink.get(), vpnUuid, vrfEntry, vpnId);
 360                 }
 361             }
 362         }
 363     }
 364
 365
 366     /*
 367       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 368       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 369       provided by ResourceBatchingManager
 370      */
 371     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 372         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 373
 374         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 375         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 376         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 377
 378         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 379         final String rd = vrfTableKey.getRouteDistinguisher();
 380         if (vpnToDpnList != null) {
 381             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 382                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 383                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 384                 }
 385             }
 386         }
 387     }
 388
 389     /*
 390      * Returns true if the specified nexthop is the other endpoint in an
 391      * InterVpnLink, regarding one of the VPN's point of view.
 392      */
 393     private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) {
 394         return
 395                 interVpnLink != null
 396                         && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
 397                         && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
 398                         || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
 399                         && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
 400     }
 401
 402
 403     // FIXME: Refactoring needed here.
 404     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 405     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 406                                    int addOrRemove) {
 407         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 408         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 409
 410         String rd = vrfTableKey.getRouteDistinguisher();
 411         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 412         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 413             if (vpnInstance == null) {
 414                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 415                 return;
 416             }
 417         } else {
 418             Preconditions.checkNotNull(vpnInstance,
 419                     "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 420         }
 421         String vpnUuid = vpnInstance.getVpnInstanceName();
 422         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 423                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 424
 425         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 426         // there an interVpnLink for the involved vpn in order to make learn the new route to
 427         // the other part of the inter-vpn-link.
 428
 429         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 430         Optional<InterVpnLink> interVpnLink =
 431                 (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 432                         : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 433         if ( !interVpnLink.isPresent() ) {
 434             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 435             return;
 436         }
 437
 438         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 439         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 440         boolean proceed = (addOrRemove == NwConstants.DEL_FLOW )
 441                 || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 442                 && interVpnLink.get().isBgpRoutesLeaking() );
 443
 444         if ( proceed ) {
 445             String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) )
 446                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 447                     : vpnUuid;
 448
 449             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 450             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 451
 452             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 453                     InstanceIdentifier.builder(FibEntries.class)
 454                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 455                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 456                             .build();
 457             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 458                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 459                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 460                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 461                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 462                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 463                         .setLabel(label)
 464                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 465                         .build();
 466                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 467             } else {
 468                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 469                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 470             }
 471         }
 472     }
 473
 474     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 475         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 476         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 477         prefixBuilder.setDpnId(lri.getDpnId());
 478         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 479         prefixBuilder.setIpAddress(lri.getPrefix());
 480         // Increment the refCount here
 481         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 482                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 483         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 484         if (!isPresentInList) {
 485             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 486             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 487             vpnInstanceNames.add(vpnInstanceName);
 488             builder.setVpnInstanceList(vpnInstanceNames);
 489             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 490         } else {
 491             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 492         }
 493         return prefixBuilder.build();
 494     }
 495
 496     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 497                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 498         Boolean wrTxPresent = true;
 499         if (tx == null) {
 500             wrTxPresent = false;
 501             tx = dataBroker.newWriteOnlyTransaction();
 502         }
 503         synchronized (vrfEntry.getLabel().toString().intern()) {
 504             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 505             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 506                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 507
 508                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 509                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 510                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 511                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 512                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 513                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 514                         }
 515                     }
 516                 }
 517                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 518                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 519             }
 520         }
 521         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 522         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 523         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 524         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 525         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 526
 527         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 528             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 529             // reinitialize instructions list for LFIB Table
 530             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 531
 532             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 533             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 534             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 535             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 536
 537             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 538         }
 539         if (!wrTxPresent ) {
 540             tx.submit();
 541         }
 542     }
 543
 544     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 545         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 546         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 547         // packet is commuted from Vpn2 to Vpn1.
 548         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 549         if ( !vpnNameOpc.isPresent() ) {
 550             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 551             return;
 552         }
 553
 554         String vpnName = vpnNameOpc.get();
 555         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 556         boolean interVpnLinkFound = false;
 557         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 558             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 559             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 560                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 561             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 562                 interVpnLinkFound = true;
 563
 564                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 565                 if ( !vpnLinkState.isPresent()
 566                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 567                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 568                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 569                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 570                     return;
 571                 }
 572
 573                 List<BigInteger> targetDpns =
 574                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
 575                                 : vpnLinkState.get().getSecondEndpointState().getDpId();
 576                 Long lportTag =
 577                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 578                                 : vpnLinkState.get().getFirstEndpointState().getLportTag();
 579
 580                 for ( BigInteger dpId : targetDpns ) {
 581                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 582
 583                     BigInteger[] metadata = new BigInteger[] {
 584                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 585                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 586                     };
 587                     List<InstructionInfo> instructions =
 588                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 589                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 590                                     new InstructionInfo(InstructionType.goto_table,
 591                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 592
 593                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 594                             NwConstants.ADD_FLOW, null);
 595                 }
 596
 597                 break;
 598             }
 599         }
 600
 601         if ( !interVpnLinkFound ) {
 602             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 603                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 604         }
 605     }
 606
 607
 608
 609     private void installRouteInInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 610                                             final VrfEntry vrfEntry, long vpnTag) {
 611         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 612         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 613                 && vrfEntry.getNextHopAddressList().size() == 1);
 614         String destination = vrfEntry.getDestPrefix();
 615         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 616
 617         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 618         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 619         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 620         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 621         if ( !interVpnLinkState.isPresent() ) {
 622             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 623             return;
 624         }
 625         if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 626             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 627                     destination, nextHop, interVpnLink.getName());
 628             return;
 629         }
 630
 631
 632         // Everything Ok
 633         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 634         List<BigInteger> targetDpns =
 635                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 636                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 637
 638         Long otherEndpointlportTag =
 639                 vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
 640                         : interVpnLinkState.get().getFirstEndpointState().getLportTag();
 641
 642         BigInteger[] metadata = new BigInteger[] {
 643                 MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag.intValue(),
 644                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 645                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 646         };
 647         int instIdx = 0;
 648         List<Instruction> instructions =
 649                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 650                         new InstructionInfo(InstructionType.goto_table,
 651                                 new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 652
 653         String values[] = destination.split("/");
 654         String destPrefixIpAddress = values[0];
 655         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 656
 657         List<MatchInfo> matches = new ArrayList<>();
 658         matches.add(new MatchInfo(MatchFieldType.metadata,
 659                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 660                         MetaDataUtil.METADATA_MASK_VRFID }));
 661         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 662
 663         if (prefixLength != 0) {
 664             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 665                     new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 666         }
 667
 668         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 669         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), destination, nextHop);
 670         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 671                 COOKIE_VM_FIB_TABLE, matches, instructions);
 672
 673         for ( BigInteger dpId : targetDpns ) {
 674             mdsalManager.installFlow(dpId, flowEntity);
 675         }
 676     }
 677
 678     private void removeRouteFromInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 679                                              final VrfEntry vrfEntry) {
 680
 681         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 682         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 683                 && vrfEntry.getNextHopAddressList().size() == 1);
 684
 685         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 686         if ( !interVpnLinkState.isPresent() ) {
 687             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 688             return;
 689         }
 690
 691         // Everything Ok
 692         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 693         List<BigInteger> targetDpns =
 694                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 695                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 696
 697         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 698         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 699         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
 700         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
 701                 .setFlowName(flowRef).build();
 702
 703         for ( BigInteger dpId : targetDpns ) {
 704             mdsalManager.removeFlow(dpId, flow);
 705         }
 706
 707     }
 708
 709     private boolean isVpnFirstEndPoint(InterVpnLink interVpnLink, String vpnName) {
 710         return interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 711     }
 712
 713     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 714                                                      InstanceIdentifier<T> path) {
 715
 716         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 717
 718         Optional<T> result = Optional.absent();
 719         try {
 720             result = tx.read(datastoreType, path).get();
 721         } catch (Exception e) {
 722             throw new RuntimeException(e);
 723         }
 724
 725         return result;
 726     }
 727
 728     private void makeSubnetRouteTableMissFlow(BigInteger dpnId, int addOrRemove) {
 729         final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 730         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 731         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 732         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[]{}));
 733         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 734         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 735         String flowRef = getTableMissFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, NwConstants.TABLE_MISS_FLOW);
 736         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef,
 737                 NwConstants.TABLE_MISS_PRIORITY, "Subnet Route Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
 738
 739         if (addOrRemove == NwConstants.ADD_FLOW) {
 740             mdsalManager.installFlow(flowEntity);
 741         } else {
 742             mdsalManager.removeFlow(flowEntity);
 743         }
 744     }
 745
 746     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 747         List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 748         Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 749         String localNextHopIP = vrfEntry.getDestPrefix();
 750
 751         if (localNextHopInfo == null) {
 752             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 753             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 754             if (extraRoute != null) {
 755                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 756                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 757                     if (nextHopIp != null) {
 758                         localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
 759                         localNextHopIP = nextHopIp + "/32";
 760                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 761                         returnLocalDpnId.add(dpnId);
 762                     }
 763                 }
 764             }
 765             if (localNextHopInfo == null) {
 766             /* imported routes case */
 767                 synchronized (vrfEntry.getLabel().toString().intern()) {
 768                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 769                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 770                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 771                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 772                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 773                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 774                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 775                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 776                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 777                                     localNextHopIP = lri.getPrefix();
 778                                 } else {
 779                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 780                                     localNextHopIP = lri.getPrefix();
 781                                 }
 782                             }
 783                             if (localNextHopInfo != null) {
 784                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 785                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 786                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 787                                 returnLocalDpnId.add(dpnId);
 788                             }
 789                         }
 790                     }
 791                 }
 792             }
 793         } else {
 794             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 795             returnLocalDpnId.add(dpnId);
 796         }
 797
 798         return returnLocalDpnId;
 799     }
 800
 801     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 802                                                 final VrfEntry vrfEntry, Long parentVpnId){
 803         if (localNextHopInfo != null) {
 804             final BigInteger dpnId = localNextHopInfo.getDpnId();
 805             if (!isVpnPresentInDpn(rd, dpnId)) {
 806                 return BigInteger.ZERO;
 807             }
 808
 809             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
 810
 811             List<ActionInfo> actionsInfos =
 812                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 813             final List<InstructionInfo> instructions =
 814                     Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 815             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 816                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 817             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 818             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 819                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 820                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 821             } else {
 822                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 823             }
 824             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 825             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 826                     new Callable<List<ListenableFuture<Void>>>() {
 827                         @Override
 828                         public List<ListenableFuture<Void>> call() throws Exception {
 829                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 830                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 831                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 832                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 833                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 834                             }
 835                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 836                             futures.add(tx.submit());
 837                             return futures;
 838                         }
 839                     });
 840             return dpnId;
 841         }
 842         return BigInteger.ZERO;
 843     }
 844
 845     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 846         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 847         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 848         if (dpnInVpn.isPresent()) {
 849             return true;
 850         }
 851         return false;
 852     }
 853
 854     private LabelRouteInfo getLabelRouteInfo(Long label) {
 855         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 856                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 857         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 858         if (opResult.isPresent()) {
 859             return opResult.get();
 860         }
 861         return null;
 862     }
 863
 864     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 865         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 866         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 867                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 868         if (lri == null) {
 869             return true;
 870         }
 871         List<String> vpnInstancesList = lri.getVpnInstanceList();
 872         if (vpnInstancesList.contains(vpnInstanceName)) {
 873             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 874             vpnInstancesList.remove(vpnInstanceName);
 875         }
 876         if (vpnInstancesList.size() == 0) {
 877             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 878             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 879             return true;
 880         } else {
 881             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 882             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 883             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 884         }
 885         return false;
 886     }
 887
 888     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 889                                       WriteTransaction tx) {
 890         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 891         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 892
 893
 894         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 895
 896         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 897                 dpId, label, groupId);
 898     }
 899
 900     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 901                                                  WriteTransaction tx) {
 902         List<MatchInfo> mkMatches = new ArrayList<>();
 903
 904         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 905
 906         // Matching metadata
 907         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 908         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 909
 910         List<InstructionInfo> mkInstructions = new ArrayList<>();
 911         mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 912
 913         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 914                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 915                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 916
 917         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 918
 919         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 920
 921         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 922         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 923                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 924                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 925         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 926     }
 927
 928     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 929         FlowEntity flowEntity;
 930         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 931         List<MatchInfo> mkMatches = new ArrayList<>();
 932         // Matching metadata
 933         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 934         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 935                 NwConstants.INTERNAL_TUNNEL_TABLE,
 936                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 937                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 938                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 939         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 940         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 941         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 942                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 943                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 944
 945         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 946         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 947     }
 948
 949     /**
 950      * Delete local FIB entry
 951      * @param vpnId
 952      * @param rd
 953      * @param vrfEntry
 954      * @return
 955      */
 956     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 957         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 958         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 959         String localNextHopIP = vrfEntry.getDestPrefix();
 960
 961         if (localNextHopInfo == null) {
 962             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 963             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 964             if (extra_route != null) {
 965                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 966                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 967                     if (nextHopIp != null) {
 968                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 969                         localNextHopIP = nextHopIp + "/32";
 970                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 971                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 972                         if (!dpnId.equals(BigInteger.ZERO)) {
 973                             returnLocalDpnId.add(dpnId);
 974                         }
 975                     }
 976                 }
 977             }
 978
 979             if (localNextHopInfo == null) {
 980               /* Imported VRF entry */
 981                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 982                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 983                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 984                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
 985                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
 986                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
 987                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 988                     if (!dpnId.equals(BigInteger.ZERO)) {
 989                         returnLocalDpnId.add(dpnId);
 990                     }
 991                 }
 992             }
 993
 994
 995         } else {
 996             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 997                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 998             if (!dpnId.equals(BigInteger.ZERO)) {
 999                 returnLocalDpnId.add(dpnId);
1000             }
1001         }
1002
1003         return returnLocalDpnId;
1004     }
1005
1006     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1007                                                 final Long vpnId, final String rd,
1008                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1009         if (localNextHopInfo != null) {
1010             final BigInteger dpnId = localNextHopInfo.getDpnId();;
1011             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1012             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
1013                     new Callable<List<ListenableFuture<Void>>>() {
1014                         @Override
1015                         public List<ListenableFuture<Void>> call() throws Exception {
1016                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1017                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1018                                     NwConstants.DEL_FLOW, tx);
1019                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1020                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1021                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1022                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1023                             }
1024                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1025                             futures.add(tx.submit());
1026                             return futures;
1027                         }
1028                     });
1029             //TODO: verify below adjacency call need to be optimized (?)
1030             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
1031             return dpnId;
1032         }
1033         return BigInteger.ZERO;
1034     }
1035
1036
1037     private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
1038         return InstanceIdentifier.builder(PrefixToInterface.class)
1039                 .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
1040     }
1041
1042     private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
1043         Optional<Prefixes> localNextHopInfoData =
1044                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
1045         return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
1046     }
1047
1048     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1049         return InstanceIdentifier.builder(VpnToExtraroute.class)
1050                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1051                         new ExtrarouteKey(ipPrefix)).build();
1052     }
1053
1054     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1055         Optional<Extraroute> extraRouteInfo =
1056                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1057         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1058
1059     }
1060
1061     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1062         try {
1063             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1064                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1065             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1066             if(!rpcResult.isSuccessful()) {
1067                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1068             } else {
1069                 return rpcResult.getResult().getTunnelType();
1070             }
1071
1072         } catch (InterruptedException | ExecutionException e) {
1073             LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
1074         }
1075
1076         return null;
1077
1078     }
1079     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1080                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1081         Boolean wrTxPresent = true;
1082         if (tx == null) {
1083             wrTxPresent = false;
1084             tx = dataBroker.newWriteOnlyTransaction();
1085         }
1086         String rd = vrfTableKey.getRouteDistinguisher();
1087         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1088                 vrfEntry.getDestPrefix(), rd, tx);
1089         /********************************************/
1090         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1091
1092         if (tunnelInterfaceList.isEmpty()) {
1093             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1094                     vrfEntry.getNextHopAddressList(), rd);
1095             LOG.warn("Failed to add Route: {} in vpn: {}",
1096                     vrfEntry.getDestPrefix(), rd);
1097             return;
1098         }
1099
1100         for (String tunnelInterface : tunnelInterfaceList) {
1101             List<InstructionInfo> instructions = new ArrayList<>();
1102             List<ActionInfo> actionInfos = new ArrayList<>();
1103             Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1104             if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1105                 LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1106                 actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
1107                 actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
1108             } else {
1109                 int label = vrfEntry.getLabel().intValue();
1110                 BigInteger tunnelId;
1111                 // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1112                 if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1113                     tunnelId = BigInteger.valueOf(label);
1114                 } else {
1115                     tunnelId = BigInteger.valueOf(label);
1116                 }
1117
1118                 LOG.debug("adding set tunnel id action for label {}", label);
1119                 actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
1120             }
1121             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
1122             if(egressActions.isEmpty()){
1123                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
1124                 return;
1125             }
1126             actionInfos.addAll(egressActions);
1127             instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
1128             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1129         }
1130         if(!wrTxPresent ){
1131             tx.submit();
1132         }
1133         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1134     }
1135
1136     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1137         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1138         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1139         if (dpnInVpn.isPresent()) {
1140             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1141             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1142
1143             if (vpnInterfaces.remove(currVpnInterface)) {
1144                 if (vpnInterfaces.isEmpty()) {
1145                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1146                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1147                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1148                 } else {
1149                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1150                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1151                             VpnInterfaces.class,
1152                             new VpnInterfacesKey(intfName)));
1153                 }
1154             }
1155         }
1156     }
1157
1158     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1159     /* Get interface info from prefix to interface mapping;
1160         Use the interface info to get the corresponding vpn interface op DS entry,
1161         remove the adjacency corresponding to this fib entry.
1162         If adjacency removed is the last adjacency, clean up the following:
1163          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1164          - prefix to interface entry
1165          - vpn interface op DS
1166      */
1167         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1168         Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1169         Extraroute extraRoute = null;
1170         if (prefixInfo == null) {
1171             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1172             if(extraRoute != null) {
1173                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1174                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1175
1176                     if (nextHopIp != null) {
1177                         prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
1178                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1179                     }
1180                 }
1181             }
1182             if (prefixInfo == null) {
1183                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1184                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1185                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1186                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1187                     prefixBuilder.setDpnId(lri.getDpnId());
1188                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1189                     prefixBuilder.setIpAddress(lri.getPrefix());
1190                     prefixInfo = prefixBuilder.build();
1191                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1192                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1193                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1194                 }
1195             }
1196         } else {
1197             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1198         }
1199     }
1200
1201     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1202                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1203
1204         if (prefixInfo == null) {
1205             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1206             return; //Don't have any info for this prefix (shouldn't happen); need to return
1207         }
1208
1209         String ifName = prefixInfo.getVpnInterfaceName();
1210         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1211         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1212                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1213     }
1214
1215     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1216         Prefixes prefixInfo;
1217         Long vpnId;
1218         String rd;
1219         VrfEntry vrfEntry;
1220         Extraroute extraRoute;
1221
1222         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1223                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1224             this.prefixInfo = prefixInfo;
1225             this.vpnId = vpnId;
1226             this.rd= rd;
1227             this.vrfEntry= vrfEntry;
1228             this.extraRoute = extraRoute;
1229         }
1230
1231         @Override
1232         public List<ListenableFuture<Void>> call() throws Exception {
1233             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1234             // to call the respective helpers.
1235             String ifName = prefixInfo.getVpnInterfaceName();
1236             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1237                     FibUtil.getVpnInterfaceIdentifier(ifName));
1238             if (optvpnInterface.isPresent()) {
1239                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1240                 if (vpnId != associatedVpnId) {
1241                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1242                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1243                     LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
1244                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1245                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1246                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1247                     return null;
1248                 } else {
1249                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1250                             vrfEntry.getDestPrefix(), associatedVpnId);
1251                 }
1252             }
1253             if (extraRoute != null) {
1254                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1255                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1256             }
1257             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1258                     FibUtil.getAdjListPath(ifName));
1259             int numAdj = 0;
1260             if (optAdjacencies.isPresent()) {
1261                 numAdj = optAdjacencies.get().getAdjacency().size();
1262             }
1263             //remove adjacency corr to prefix
1264             if (numAdj > 1) {
1265                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1266                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1267                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1268             }
1269             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1270                 //clean up the vpn interface from DpnToVpn list
1271                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1272                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1273                         FibUtil.getVpnInterfaceIdentifier(ifName));
1274             }
1275
1276             synchronized (vrfEntry.getLabel().toString().intern()) {
1277                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1278                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1279                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1280                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1281                     String vpnInstanceName = "";
1282                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1283                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1284                     }
1285                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1286                     if (lriRemoved) {
1287                         String parentRd = lri.getParentVpnRd();
1288                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1289                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1290                     }
1291                 } else {
1292                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1293                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1294                 }
1295             }
1296             return null;
1297         }
1298     }
1299
1300     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1301         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1302         final String rd  = vrfTableKey.getRouteDistinguisher();
1303         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1304         if (vpnInstance == null) {
1305             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1306             return;
1307         }
1308         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1309         long elanTag = 0L;
1310         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1311         if (subnetRoute != null) {
1312             elanTag = subnetRoute.getElantag();
1313             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1314                     rd, vrfEntry.getDestPrefix(), elanTag);
1315             if (vpnToDpnList != null) {
1316                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1317                 dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1318                         new Callable<List<ListenableFuture<Void>>>() {
1319                             @Override
1320                             public List<ListenableFuture<Void>> call() throws Exception {
1321                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1322
1323                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1324
1325                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1326                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1327                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1328                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1329                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1330                                     }
1331                                 }
1332                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1333                                 futures.add(tx.submit());
1334                                 return futures;
1335                             }
1336                         });
1337             }
1338             synchronized (vrfEntry.getLabel().toString().intern()) {
1339                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1340                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1341                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1342                     String vpnInstanceName = "";
1343                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1344                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1345                     }
1346                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1347                     if (lriRemoved) {
1348                         String parentRd = lri.getParentVpnRd();
1349                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1350                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1351                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1352                                 vrfEntry.getDestPrefix());
1353                     }
1354                 } else {
1355                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1356                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1357                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1358                             vrfEntry.getDestPrefix());
1359                 }
1360             }
1361             return;
1362         }
1363
1364         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1365                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1366         if (vpnToDpnList != null) {
1367             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1368             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1369                     new Callable<List<ListenableFuture<Void>>>() {
1370                         @Override
1371                         public List<ListenableFuture<Void>> call() throws Exception {
1372                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1373
1374                             if (localDpnIdList.size() <= 0) {
1375                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1376                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1377                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1378                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1379                                         }
1380                                     } else {
1381                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1382                                     }
1383                                 }
1384                             } else {
1385                                 for (BigInteger localDpnId : localDpnIdList) {
1386                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1387                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1388                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1389                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1390                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1391                                                 }
1392                                             } else {
1393                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1394                                             }
1395                                         }
1396                                     }
1397                                 }
1398                             }
1399                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1400                             futures.add(tx.submit());
1401                             return futures;
1402                         }
1403                     });
1404         }
1405
1406         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1407         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1408         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1409
1410         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1411         // of the interVpnLink.
1412         Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
1413         if ( vpnUuid.isPresent() ) {
1414             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, vpnUuid.get());
1415             String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
1416
1417             if ( interVpnLink.isPresent()
1418                     && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
1419                     && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
1420                     || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
1421                     && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
1422                 // This is route that points to the other endpoint of an InterVpnLink
1423                 // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1424                 removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
1425             }
1426         }
1427
1428     }
1429
1430     /*
1431       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1432       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1433       provided by ResourceBatchingManager
1434      */
1435     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1436         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1437
1438         final String rd  = vrfTableKey.getRouteDistinguisher();
1439         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1440         if (vpnInstance == null) {
1441             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1442             return;
1443         }
1444         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1445         if (vpnToDpnList != null) {
1446             for (VpnToDpnList curDpn : vpnToDpnList) {
1447                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1448                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1449                 }
1450             }
1451         }
1452     }
1453
1454     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1455                                   final long vpnId, final VrfTablesKey vrfTableKey,
1456                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1457
1458         Boolean wrTxPresent = true;
1459         if (tx == null) {
1460             wrTxPresent = false;
1461             tx = dataBroker.newWriteOnlyTransaction();
1462         }
1463
1464         LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1465         String rd = vrfTableKey.getRouteDistinguisher();
1466
1467         if(localDpnId != null) {
1468             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1469             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1470             return;
1471         }
1472
1473         // below two reads are kept as is, until best way is found to identify dpnID
1474         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1475         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1476
1477         if (localNextHopInfo == null && extraRoute != null) {
1478             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1479             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1480                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1481                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1482             }
1483         } else {
1484             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1485         }
1486         if(!wrTxPresent ){
1487             tx.submit();
1488         }
1489     }
1490
1491     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1492                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1493         boolean isRemoteRoute = true;
1494         if (localNextHopInfo != null) {
1495             isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId()));
1496         }
1497         if (isRemoteRoute) {
1498             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1499             return true;
1500         } else {
1501             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1502             return false;
1503         }
1504     }
1505
1506     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1507         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1508         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1509     }
1510
1511     private long get
1512             (byte[] rawIpAddress) {
1513         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1514                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1515     }
1516
1517     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1518                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1519         Boolean wrTxPresent = true;
1520         if (tx == null) {
1521             wrTxPresent = false;
1522             tx = dataBroker.newWriteOnlyTransaction();
1523         }
1524
1525         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1526         String values[] = vrfEntry.getDestPrefix().split("/");
1527         String ipAddress = values[0];
1528         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1529         if (addOrRemove == NwConstants.ADD_FLOW) {
1530             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1531         } else {
1532             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1533         }
1534         InetAddress destPrefix;
1535         try {
1536             destPrefix = InetAddress.getByName(ipAddress);
1537         } catch (UnknownHostException e) {
1538             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1539             return;
1540         }
1541
1542         List<MatchInfo> matches = new ArrayList<>();
1543
1544         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1545                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1546
1547         matches.add(new MatchInfo(MatchFieldType.eth_type,
1548                 new long[] { NwConstants.ETHTYPE_IPV4 }));
1549
1550         if(prefixLength != 0) {
1551             matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1552                     destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1553         }
1554         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1555         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1556         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1557                 COOKIE_VM_FIB_TABLE, matches, instructions);
1558
1559         Flow flow = flowEntity.getFlowBuilder().build();
1560         String flowId = flowEntity.getFlowId();
1561         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1562         Node nodeDpn = buildDpnNode(dpId);
1563
1564         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1565                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1566                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1567         if (addOrRemove == NwConstants.ADD_FLOW) {
1568             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1569         } else {
1570             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1571         }
1572
1573         if(!wrTxPresent ){
1574             tx.submit();
1575         }
1576     }
1577
1578     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1579     private Node buildDpnNode(BigInteger dpnId) {
1580         NodeId nodeId = new NodeId("openflow:" + dpnId);
1581         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1582
1583         return nodeDpn;
1584     }
1585
1586     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1587                                     int addOrRemove, WriteTransaction tx) {
1588         Boolean wrTxPresent = true;
1589         if (tx == null) {
1590             wrTxPresent = false;
1591             tx = dataBroker.newWriteOnlyTransaction();
1592         }
1593
1594         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1595         matches.add(new MatchInfo(MatchFieldType.eth_type,
1596                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1597         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1598
1599         // Install the flow entry in L3_LFIB_TABLE
1600         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1601
1602         FlowEntity flowEntity;
1603         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1604                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1605         Flow flow = flowEntity.getFlowBuilder().build();
1606         String flowId = flowEntity.getFlowId();
1607         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1608         Node nodeDpn = buildDpnNode(dpId);
1609         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1610                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1611                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1612
1613         if (addOrRemove == NwConstants.ADD_FLOW) {
1614             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1615         } else {
1616             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1617         }
1618         if(!wrTxPresent ){
1619             tx.submit();
1620         }
1621         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
1622                 dpId, label, instructions );
1623     }
1624
1625     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
1626         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1627         try {
1628             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
1629         } catch (NullPointerException e) {
1630             LOG.trace("", e);
1631         }
1632     }
1633
1634     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1635                                     final FutureCallback<List<Void>> callback) {
1636         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1637         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1638         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1639         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1640         if (vrfTable.isPresent()) {
1641             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1642             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1643                     new Callable<List<ListenableFuture<Void>>>() {
1644                         @Override
1645                         public List<ListenableFuture<Void>> call() throws Exception {
1646                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1647                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1648                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1649                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1650
1651                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1652                                     if (subnetRoute != null) {
1653                                         long elanTag = subnetRoute.getElantag();
1654                                         installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1655                                         continue;
1656                                     }
1657                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1658                                         //Handle local flow creation for imports
1659                                         LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1660                                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()
1661                                         ) && vrfEntry.getNextHopAddressList()
1662                                                 .contains(lri.getNextHopIpList().get(0))) {
1663                                             if (lri.getDpnId().equals(dpnId)) {
1664                                                 createLocalFibEntry(vpnId, rd, vrfEntry);
1665                                                 continue;
1666                                             }
1667                                         }
1668                                     }
1669                                     // Passing null as we don't know the dpn
1670                                     // to which prefix is attached at this point
1671                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1672                                 }
1673                                 //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1674                                 futures.add(tx.submit());
1675                                 if (callback != null) {
1676                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1677                                     Futures.addCallback(listenableFuture, callback);
1678                                 }
1679                             }
1680                             return futures;
1681                         }
1682                     });
1683         }
1684     }
1685
1686     public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1687                                  final String localNextHopIp, final String remoteNextHopIp) {
1688         LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1689                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1690         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1691         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1692         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1693         if (vrfTable.isPresent()) {
1694             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1695             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1696                     new Callable<List<ListenableFuture<Void>>>() {
1697                         @Override
1698                         public List<ListenableFuture<Void>> call() throws Exception {
1699                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1700                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1701                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1702                                 LOG.trace("populate FIB starts on Dpn " + dpnId
1703                                         + "rd  " + rd.toString()
1704                                         + "localNextHopIp " + localNextHopIp
1705                                         + "remoteNextHopIp" + remoteNextHopIp
1706                                         + "vpnId " + vpnId );
1707                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1708                                     LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1709
1710                                     if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1711                                         if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1712                                             LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1713                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1714                                         }
1715                                     } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
1716                                         BigInteger dpnIdForPrefix = null;
1717                                         String destPfx = vrfEntry.getDestPrefix();
1718                                         if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
1719                                             Optional<Extraroute> extraRouteInfo =
1720                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1721                                                             getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1722                                             if (extraRouteInfo.isPresent()) {
1723                                                 continue;
1724                                             }
1725                                             dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1726                                         } else {
1727                                             // Subnet Route handling
1728                                             Optional<Prefixes> localNextHopInfoData =
1729                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1730                                                             FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
1731                                             if (localNextHopInfoData.isPresent()) {
1732                                                 Prefixes prefixes = localNextHopInfoData.get();
1733                                                 dpnIdForPrefix = prefixes.getDpnId();
1734                                             }
1735                                         }
1736                                         if (dpnIdForPrefix == null) {
1737                                             LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1738                                                     vrfEntry.getDestPrefix());
1739                                             continue;
1740                                         }
1741                                         int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1742                                         if (sameDpnId != 0) {
1743                                             LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1744                                                     dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1745                                             continue;
1746                                         }
1747                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1748                                         List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
1749                                         newNextHopAddrList.add(localNextHopIp);
1750                                         VrfEntry newVrfEntry =
1751                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
1752                                         // Just update the VrfEntry
1753                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1754                                                 vrfEntryId, newVrfEntry);
1755                                         // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
1756                                         //       vrfEntryId, newVrfEntry);
1757                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1758                                         LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1759                                     }
1760                                 }
1761                                 futures.add(writeTransaction.submit());
1762                                 LOG.trace("populate FIB ends on Dpn " + dpnId
1763                                         + "rd  " + rd.toString()
1764                                         + "localNextHopIp " + localNextHopIp
1765                                         + "remoteNextHopIp" + remoteNextHopIp
1766                                         + "vpnId " + vpnId);
1767                             }
1768                             return futures;
1769                         }
1770                     });
1771         }
1772     }
1773
1774     public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
1775                                   final long vpnId, final String  rd, final String destPrefix ,
1776                                   final String localNextHopIP, final String remoteNextHopIp) {
1777
1778         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1779         dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1780                         + "local dpid" + localDpnId
1781                         + "remote dpid" + remoteDpnId
1782                         + "vpnId" + vpnId
1783                         + "localNHIp" + localNextHopIP
1784                         + "remoteNHIp" + remoteNextHopIp,
1785                 new Callable<List<ListenableFuture<Void>>>() {
1786                     @Override
1787                     public List<ListenableFuture<Void>> call() throws Exception {
1788                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1789                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1790                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1791                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1792                         if (vrfEntry == null)
1793                             return futures;
1794                         LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1795                                         "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1796                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1797                         if (action == true) {
1798                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1799                             LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1800                             createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1801                         } else {
1802                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1803                             LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1804                             deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1805                         }
1806                         futures.add(writeTransaction.submit());
1807                         return futures;
1808                     }
1809                 });
1810     }
1811
1812     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1813                                  final FutureCallback<List<Void>> callback) {
1814         LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1815         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1816         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1817         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1818         if (vrfTable.isPresent()) {
1819             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1820             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1821                     new Callable<List<ListenableFuture<Void>>>() {
1822                         @Override
1823                         public List<ListenableFuture<Void>> call() throws Exception {
1824                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1825                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1826                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1827                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1828                                     /* Handle subnet routes here */
1829                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1830                                     if (subnetRoute != null) {
1831                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1832                                                 dpnId, rd);
1833                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1834                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1835                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1836                                                 vrfEntry.getDestPrefix());
1837                                         continue;
1838                                     }
1839                                     // Passing null as we don't know the dpn
1840                                     // to which prefix is attached at this point
1841                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1842                                 }
1843                                 futures.add(tx.submit());
1844                                 if (callback != null) {
1845                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1846                                     Futures.addCallback(listenableFuture, callback);
1847                                 }
1848                             }
1849                             return futures;
1850                         }
1851
1852                     });
1853         }
1854     }
1855
1856     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1857                                  final String localNextHopIp, final String remoteNextHopIp,
1858                                  final FutureCallback<List<Void>> callback) {
1859         LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1860                         " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1861                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1862         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1863         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1864         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1865         if (vrfTable.isPresent()) {
1866             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1867             dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
1868                     new Callable<List<ListenableFuture<Void>>>() {
1869                         @Override
1870                         public List<ListenableFuture<Void>> call() throws Exception {
1871                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1872                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1873                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1874                                 LOG.trace("cleanup FIB starts on Dpn " + dpnId
1875                                         + "rd  " + rd.toString()
1876                                         + "localNextHopIp " + localNextHopIp
1877                                         + "remoteNextHopIp" + remoteNextHopIp
1878                                         + "vpnId " + vpnId);
1879
1880                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1881                                     LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1882                                     if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1883                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1884                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1885                                     }
1886
1887                                     if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1888                                         LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1889                                                 vrfEntry.getDestPrefix(), dpnId);
1890                                         String destPfx = vrfEntry.getDestPrefix();
1891                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1892                                         List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1893                                         newList.remove(localNextHopIp);
1894                                         VrfEntry newVrfEntry =
1895                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1896                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1897                                                 vrfEntryId, newVrfEntry);
1898                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1899                                         LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1900                                     }
1901                                 }
1902                                 futures.add(writeTransaction.submit());
1903                                 if (callback != null) {
1904                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1905                                     Futures.addCallback(listenableFuture, callback);
1906                                 }
1907                                 LOG.trace("cleanup FIB ends on Dpn " + dpnId
1908                                         + "rd  " + rd.toString()
1909                                         + "localNextHopIp " + localNextHopIp
1910                                         + "remoteNextHopIp" + remoteNextHopIp
1911                                         + "vpnId " + vpnId);
1912                             }
1913                             return futures;
1914                         }
1915                     });
1916
1917         }
1918     }
1919
1920     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1921         InstanceIdentifierBuilder<VrfTables> idBuilder =
1922                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1923         InstanceIdentifier<VrfTables> id = idBuilder.build();
1924         return id;
1925     }
1926
1927     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1928         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1929                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1930                 .append(priority).toString();
1931     }
1932
1933     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1934         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1935                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1936                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1937                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1938                 .append(destPrefix.getHostAddress()).toString();
1939     }
1940
1941     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1942         return new StringBuilder(64).append(FLOWID_PREFIX)
1943                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1944                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1945                 .append(nextHop).toString();
1946     }
1947
1948     protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
1949                                             final VrfEntry vrfEntry, String rd) {
1950         List<String> adjacencyList = new ArrayList<>();
1951         List<String> prefixIpList = new ArrayList<>();
1952         LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
1953                 remoteDpnId, vpnId, vrfEntry);
1954         try {
1955             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1956                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1957                 if (extra_route == null) {
1958                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1959                 } else {
1960                     prefixIpList = new ArrayList<>();
1961                     for (String extraRouteIp : extra_route.getNexthopIpList()) {
1962                         prefixIpList.add(extraRouteIp + "/32");
1963                     }
1964                 }
1965             } else {
1966                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1967             }
1968
1969             for (String prefixIp : prefixIpList) {
1970                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
1971                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
1972                     String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
1973                             prefixIp, nextHopIp);
1974                     if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
1975                         adjacencyList.add(adjacency);
1976                     }
1977                 }
1978             }
1979         } catch (NullPointerException e) {
1980             LOG.trace("", e);
1981         }
1982         return adjacencyList;
1983     }
1984
1985     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
1986         InstanceIdentifier<VpnInstanceOpDataEntry> id =
1987                 InstanceIdentifier.create(VpnInstanceOpData.class)
1988                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
1989         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
1990                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1991         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
1992     }
1993
1994     public void processNodeAdd(BigInteger dpnId) {
1995         LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId);
1996         makeTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1997         makeL3IntfTblMissFlow(dpnId, NwConstants.ADD_FLOW);
1998         makeSubnetRouteTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1999     }
2000
2001     private void makeTableMissFlow(BigInteger dpnId, int addOrRemove) {
2002         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2003         // Instruction to goto L3 InterfaceTable
2004         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2005         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE }));
2006         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2007         FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_LFIB_TABLE,
2008                 getTableMissFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW),
2009                 NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
2010
2011         FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE,
2012                 getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE,
2013                         NwConstants.TABLE_MISS_FLOW),
2014                 NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow",
2015                 0, 0, COOKIE_VM_FIB_TABLE,
2016                 matches, instructions);
2017
2018         if (addOrRemove == NwConstants.ADD_FLOW) {
2019             LOG.debug("Invoking MDSAL to install Table Miss Entries");
2020             mdsalManager.installFlow(flowEntityLfib);
2021             mdsalManager.installFlow(flowEntityFib);
2022         } else {
2023             mdsalManager.removeFlow(flowEntityLfib);
2024             mdsalManager.removeFlow(flowEntityFib);
2025
2026         }
2027     }
2028
2029     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2030         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2031                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2032                 .append(FLOWID_PREFIX).toString();
2033     }
2034
2035     /*
2036      * Install flow entry in protocol table to forward mpls
2037      * coming through gre tunnel to LFIB table.
2038      */
2039     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2040         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2041         // Instruction to goto L3 InterfaceTable
2042         List<InstructionInfo> instructions = new ArrayList<>();
2043         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2044         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2045         matches.add(new MatchInfo(MatchFieldType.eth_type,
2046                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2047         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2048                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2049                         NwConstants.L3_LFIB_TABLE),
2050                 DEFAULT_FIB_FLOW_PRIORITY,
2051                 "Protocol Table For LFIB",
2052                 0, 0,
2053                 COOKIE_PROTOCOL_TABLE,
2054                 matches, instructions);
2055
2056         if (addOrRemove == NwConstants.ADD_FLOW) {
2057             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2058             mdsalManager.installFlow(flowEntityToLfib);
2059         } else {
2060             mdsalManager.removeFlow(flowEntityToLfib);
2061         }
2062     }
2063
2064     public List<String> printFibEntries() {
2065         List<String> result = new ArrayList<String>();
2066         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2067         result.add("-------------------------------------------------------------------");
2068         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2069         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2070         if (fibEntries.isPresent()) {
2071             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2072             for (VrfTables vrfTable : vrfTables) {
2073                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2074                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2075                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2076                                 vrfTable.getRouteDistinguisher(),
2077                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2078                     }
2079                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2080                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2081                                 vrfTable.getRouteDistinguisher(),
2082                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2083                     }
2084                 }
2085             }
2086         }
2087         return result;
2088     }
2089
2090     private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) {
2091         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2092         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2093         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2094         // Instruction to goto L3 InterfaceTable
2095
2096         List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
2097         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
2098                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
2099         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2100         //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
2101
2102         FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
2103                 getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
2104                 NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS,
2105                 matches, instructions);
2106         if (addOrRemove == NwConstants.ADD_FLOW) {
2107             LOG.debug("Invoking MDSAL to install L3 interface Table Miss Entries");
2108             mdsalManager.installFlow(flowEntityL3Intf);
2109         } else {
2110             mdsalManager.removeFlow(flowEntityL3Intf);
2111         }
2112     }
2113
2114     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2115         InstanceIdentifier<VrfEntry> vrfEntryId =
2116                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2117                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2118         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2119         if (vrfEntry.isPresent())  {
2120             return (vrfEntry.get());
2121         }
2122         return null;
2123     }
2124
2125     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2126         InstanceIdentifier<VrfEntry> vrfEntryId =
2127                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2128                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2129         return vrfEntryId;
2130     }
2131 }