vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.CheckedFuture;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import java.math.BigInteger;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import java.util.concurrent.LinkedBlockingQueue;
  28 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  29 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  33 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  34 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  35 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  36 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  37 import org.opendaylight.genius.mdsalutil.ActionInfo;
  38 import org.opendaylight.genius.mdsalutil.ActionType;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  41 import org.opendaylight.genius.mdsalutil.InstructionType;
  42 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  43 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  44 import org.opendaylight.genius.mdsalutil.MatchInfo;
  45 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  46 import org.opendaylight.genius.mdsalutil.NwConstants;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.genius.utils.ServiceIndex;
  49 import org.opendaylight.genius.utils.batching.ActionableResource;
  50 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  51 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  52 import org.opendaylight.genius.utils.batching.ResourceHandler;
  53 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 111 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 112 import org.opendaylight.yangtools.yang.binding.DataObject;
 113 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 114 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 115 import org.opendaylight.yangtools.yang.common.RpcResult;
 116 import org.slf4j.Logger;
 117 import org.slf4j.LoggerFactory;
 118
 119 public class VrfEntryListener extends AbstractDataChangeListener<VrfEntry> implements AutoCloseable, ResourceHandler {
 120     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 121     private static final String FLOWID_PREFIX = "L3.";
 122     private ListenerRegistration<DataChangeListener> listenerRegistration;
 123     private final DataBroker dataBroker;
 124     private final IMdsalApiManager mdsalManager;
 125     private IVpnManager vpnmanager;
 126     private NexthopManager nextHopManager;
 127     private ItmRpcService itmManager;
 128     private OdlInterfaceRpcService interfaceManager;
 129     private IdManagerService idManager;
 130     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 131     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 132     private static final int LFIB_INTERVPN_PRIORITY = 1;
 133     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 134     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 135     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 136     private static final int PERIODICITY = 500;
 137     private static Integer batchSize;
 138     private static Integer batchInterval;
 139     private static final int BATCH_SIZE = 1000;
 140     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 141     private ResourceBatchingManager resourceBatchingManager;
 142
 143     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 144                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 145                             final IdManagerService idManager) {
 146         super(VrfEntry.class);
 147         this.dataBroker = dataBroker;
 148         this.mdsalManager = mdsalApiManager;
 149         this.nextHopManager = nexthopManager;
 150         this.interfaceManager = interfaceManager;
 151         this.idManager = idManager;
 152         batchSize = Integer.getInteger("batch.size");
 153         if (batchSize == null) {
 154             batchSize = BATCH_SIZE;
 155         }
 156         batchInterval = Integer.getInteger("batch.wait.time");
 157         if (batchInterval == null) {
 158             batchInterval = PERIODICITY;
 159         }
 160         resourceBatchingManager = ResourceBatchingManager.getInstance();
 161         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 162     }
 163
 164     public void start() {
 165         LOG.info("{} start", getClass().getSimpleName());
 166         listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 167                 getWildCardPath(), this, DataChangeScope.SUBTREE);
 168     }
 169
 170     private InstanceIdentifier<VrfEntry> getWildCardPath() {
 171         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 172     }
 173
 174     @Override
 175     public void close() throws Exception {
 176         if (listenerRegistration != null) {
 177             listenerRegistration.close();
 178             listenerRegistration = null;
 179         }
 180         LOG.info("{} close", getClass().getSimpleName());
 181     }
 182
 183     @Override
 184     public DataBroker getResourceBroker() {
 185         return dataBroker;
 186     }
 187
 188     @Override
 189     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 190         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 191         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 192         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 193                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 194         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 195             createFibEntries(identifier, vrfEntry);
 196         } else {
 197             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 198             actResource.setAction(ActionableResource.CREATE);
 199             actResource.setInstanceIdentifier(identifier);
 200             actResource.setInstance(vrfEntry);
 201             vrfEntryBufferQ.add(actResource);
 202             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 203         }
 204         LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 205                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 206     }
 207
 208     @Override
 209     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 210         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 211         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 212         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 213                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 214         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 215             deleteFibEntries(identifier, vrfEntry);
 216         } else {
 217             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 218             actResource.setAction(ActionableResource.DELETE);
 219             actResource.setInstanceIdentifier(identifier);
 220             actResource.setInstance(vrfEntry);
 221             vrfEntryBufferQ.add(actResource);
 222             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 223         }
 224         LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 225                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 226         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 227     }
 228
 229     @Override
 230     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 231         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 232         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 233         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 234                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 235         if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 236             createFibEntries(identifier, update);
 237         } else {
 238             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 239             actResource.setAction(ActionableResource.UPDATE);
 240             actResource.setInstanceIdentifier(identifier);
 241             actResource.setInstance(update);
 242             actResource.setOldInstance(original);
 243             vrfEntryBufferQ.add(actResource);
 244         }
 245         LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 246                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 247     }
 248
 249     @Override
 250     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 251         if (vrfEntry instanceof VrfEntry) {
 252             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 253         }
 254     }
 255
 256     @Override
 257     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 258         if (vrfEntry instanceof VrfEntry) {
 259             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 260         }
 261     }
 262
 263     @Override
 264     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 265                        Object update) {
 266         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 267             createFibEntries(tx, identifier, (VrfEntry)update);
 268         }
 269     }
 270
 271     @Override
 272     public int getBatchSize() {
 273         return batchSize;
 274     }
 275
 276     @Override
 277     public int getBatchInterval() {
 278         return batchInterval;
 279     }
 280
 281     @Override
 282     public LogicalDatastoreType getDatastoreType() {
 283         return LogicalDatastoreType.CONFIGURATION;
 284     }
 285
 286     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 287         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 288
 289         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 290         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 291         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 292
 293         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 294         final Long vpnId = vpnInstance.getVpnId();
 295         final String rd = vrfTableKey.getRouteDistinguisher();
 296         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 297         if (subnetRoute != null) {
 298             final long elanTag = subnetRoute.getElantag();
 299             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 300                     rd, vrfEntry.getDestPrefix(), elanTag);
 301             if (vpnToDpnList != null) {
 302                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 303                 dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 304                         new Callable<List<ListenableFuture<Void>>>() {
 305                             @Override
 306                             public List<ListenableFuture<Void>> call() throws Exception {
 307                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 308                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 309                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 310                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 311                                     }
 312                                 }
 313                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 314                                 futures.add(tx.submit());
 315                                 return futures;
 316                             }
 317                         });
 318             }
 319             return;
 320         }
 321
 322         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 323             // When it is a leaked route, the LFIB and FIB goes a bit different.
 324             installInterVpnRouteInLFib(rd, vrfEntry);
 325             return;
 326         }
 327
 328         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 329
 330         if (vpnToDpnList != null) {
 331             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 332             dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 333                     new Callable<List<ListenableFuture<Void>>>() {
 334                         @Override
 335                         public List<ListenableFuture<Void>> call() throws Exception {
 336                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 337                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 338                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 339                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 340                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 341                                     }
 342                                 }
 343                             }
 344                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 345                             futures.add(tx.submit());
 346                             return futures;
 347                         }
 348                     });
 349         }
 350
 351         Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 352         if ( vpnUuid.isPresent() ) {
 353             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, vpnUuid.get());
 354             if ( interVpnLink.isPresent() ) {
 355                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 356                 if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid.get(), interVpnLink.get()) ) {
 357                     // This is an static route that points to the other endpoint of an InterVpnLink
 358                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 359                     installRouteInInterVpnLink(interVpnLink.get(), rd, vrfEntry, vpnId);
 360                 }
 361             }
 362         }
 363     }
 364
 365
 366     /*
 367       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 368       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 369       provided by ResourceBatchingManager
 370      */
 371     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 372         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 373
 374         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 375         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 376         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 377
 378         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 379         final String rd = vrfTableKey.getRouteDistinguisher();
 380         if (vpnToDpnList != null) {
 381             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 382                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 383                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 384                 }
 385             }
 386         }
 387     }
 388
 389     /*
 390      * Returns true if the specified nexthop is the other endpoint in an
 391      * InterVpnLink, regarding one of the VPN's point of view.
 392      */
 393     private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) {
 394         return
 395                 interVpnLink != null
 396                         && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
 397                         && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
 398                         || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
 399                         && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
 400     }
 401
 402
 403     // FIXME: Refactoring needed here.
 404     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 405     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 406                                    int addOrRemove) {
 407         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 408         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 409
 410         String rd = vrfTableKey.getRouteDistinguisher();
 411         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 412         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 413             if (vpnInstance == null) {
 414                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 415                 return;
 416             }
 417         } else {
 418             Preconditions.checkNotNull(vpnInstance,
 419                     "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 420         }
 421         String vpnUuid = vpnInstance.getVpnInstanceName();
 422         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 423                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 424
 425         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 426         // there an interVpnLink for the involved vpn in order to make learn the new route to
 427         // the other part of the inter-vpn-link.
 428
 429         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 430         Optional<InterVpnLink> interVpnLink =
 431                 (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 432                         : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 433         if ( !interVpnLink.isPresent() ) {
 434             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 435             return;
 436         }
 437
 438         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 439         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 440         boolean proceed = (addOrRemove == NwConstants.DEL_FLOW )
 441                 || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 442                 && interVpnLink.get().isBgpRoutesLeaking() );
 443
 444         if ( proceed ) {
 445             String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) )
 446                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 447                     : vpnUuid;
 448
 449             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 450             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 451
 452             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 453                     InstanceIdentifier.builder(FibEntries.class)
 454                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 455                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 456                             .build();
 457             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 458                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 459                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 460                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 461                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 462                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 463                         .setLabel(label)
 464                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 465                         .build();
 466                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 467             } else {
 468                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 469                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 470             }
 471         }
 472     }
 473
 474     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 475         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 476         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 477         prefixBuilder.setDpnId(lri.getDpnId());
 478         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 479         prefixBuilder.setIpAddress(lri.getPrefix());
 480         // Increment the refCount here
 481         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 482                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 483         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 484         if (!isPresentInList) {
 485             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 486             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 487             vpnInstanceNames.add(vpnInstanceName);
 488             builder.setVpnInstanceList(vpnInstanceNames);
 489             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 490         } else {
 491             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 492         }
 493         return prefixBuilder.build();
 494     }
 495
 496     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 497                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 498         Boolean wrTxPresent = true;
 499         if (tx == null) {
 500             wrTxPresent = false;
 501             tx = dataBroker.newWriteOnlyTransaction();
 502         }
 503         synchronized (vrfEntry.getLabel().toString().intern()) {
 504             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 505             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 506                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 507
 508                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 509                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 510                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 511                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 512                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 513                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 514                         }
 515                     }
 516                 }
 517                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 518                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 519             }
 520         }
 521         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 522         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 523         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 524         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 525         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 526
 527         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 528             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 529             // reinitialize instructions list for LFIB Table
 530             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 531
 532             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 533             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 534             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 535             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 536
 537             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 538         }
 539         if (!wrTxPresent ) {
 540             tx.submit();
 541         }
 542     }
 543
 544     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 545         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 546         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 547         // packet is commuted from Vpn2 to Vpn1.
 548         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 549         if ( !vpnNameOpc.isPresent() ) {
 550             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 551             return;
 552         }
 553
 554         String vpnName = vpnNameOpc.get();
 555         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 556         boolean interVpnLinkFound = false;
 557         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 558             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 559             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 560                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 561             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 562                 interVpnLinkFound = true;
 563
 564                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 565                 if ( !vpnLinkState.isPresent()
 566                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 567                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 568                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 569                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 570                     return;
 571                 }
 572
 573                 List<BigInteger> targetDpns =
 574                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
 575                                 : vpnLinkState.get().getSecondEndpointState().getDpId();
 576                 int lportTag =
 577                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 578                                 : vpnLinkState.get().getFirstEndpointState().getLportTag();
 579
 580                 for ( BigInteger dpId : targetDpns ) {
 581                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 582
 583                     BigInteger[] metadata = new BigInteger[] {
 584                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag, ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 585                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 586                     };
 587                     List<InstructionInfo> instructions =
 588                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 589                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 590                                     new InstructionInfo(InstructionType.goto_table,
 591                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 592
 593                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 594                             NwConstants.ADD_FLOW, null);
 595                 }
 596
 597                 break;
 598             }
 599         }
 600
 601         if ( !interVpnLinkFound ) {
 602             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 603                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 604         }
 605     }
 606
 607
 608
 609     private void installRouteInInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 610                                             final VrfEntry vrfEntry, long vpnTag) {
 611         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 612         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 613                 && vrfEntry.getNextHopAddressList().size() == 1);
 614
 615         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 616         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 617         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 618         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 619         if ( !interVpnLinkState.isPresent() ) {
 620             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 621             return;
 622         }
 623         if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 624             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 625                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList().get(0), interVpnLink.getName());
 626             return;
 627         }
 628
 629
 630         // Everything Ok
 631         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 632         List<BigInteger> targetDpns =
 633                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 634                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 635
 636         Integer otherEndpointlportTag =
 637                 vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
 638                         : interVpnLinkState.get().getFirstEndpointState().getLportTag();
 639
 640         BigInteger[] metadata = new BigInteger[] {
 641                 MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag,
 642                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 643                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 644         };
 645         List<Instruction> instructions =
 646                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 647                         new InstructionInfo(InstructionType.goto_table,
 648                                 new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 649
 650         String values[] = vrfEntry.getDestPrefix().split("/");
 651         String destPrefixIpAddress = values[0];
 652         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 653
 654         List<MatchInfo> matches = new ArrayList<>();
 655         matches.add(new MatchInfo(MatchFieldType.metadata,
 656                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 657                         MetaDataUtil.METADATA_MASK_VRFID }));
 658         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 659
 660         if (prefixLength != 0) {
 661             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 662                     new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 663         }
 664
 665         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 666         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 667         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 668         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 669                 COOKIE_VM_FIB_TABLE, matches, instructions);
 670
 671         for ( BigInteger dpId : targetDpns ) {
 672             mdsalManager.installFlow(dpId, flowEntity);
 673         }
 674     }
 675
 676     private void removeRouteFromInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 677                                              final VrfEntry vrfEntry) {
 678
 679         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 680         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 681                 && vrfEntry.getNextHopAddressList().size() == 1);
 682
 683         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 684         if ( !interVpnLinkState.isPresent() ) {
 685             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 686             return;
 687         }
 688
 689         // Everything Ok
 690         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 691         List<BigInteger> targetDpns =
 692                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 693                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 694
 695         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 696         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 697         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
 698         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
 699                 .setFlowName(flowRef).build();
 700
 701         for ( BigInteger dpId : targetDpns ) {
 702             mdsalManager.removeFlow(dpId, flow);
 703         }
 704
 705     }
 706
 707     private boolean isVpnFirstEndPoint(InterVpnLink interVpnLink, String vpnName) {
 708         return interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 709     }
 710
 711     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 712                                                      InstanceIdentifier<T> path) {
 713
 714         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 715
 716         Optional<T> result = Optional.absent();
 717         try {
 718             result = tx.read(datastoreType, path).get();
 719         } catch (Exception e) {
 720             throw new RuntimeException(e);
 721         }
 722
 723         return result;
 724     }
 725
 726     private void makeSubnetRouteTableMissFlow(BigInteger dpnId, int addOrRemove) {
 727         final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 728         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 729         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 730         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[]{}));
 731         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 732         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 733         String flowRef = getTableMissFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, NwConstants.TABLE_MISS_FLOW);
 734         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef,
 735                 NwConstants.TABLE_MISS_PRIORITY, "Subnet Route Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
 736
 737         if (addOrRemove == NwConstants.ADD_FLOW) {
 738             mdsalManager.installFlow(flowEntity);
 739         } else {
 740             mdsalManager.removeFlow(flowEntity);
 741         }
 742     }
 743
 744     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 745         List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 746         Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 747         String localNextHopIP = vrfEntry.getDestPrefix();
 748
 749         if (localNextHopInfo == null) {
 750             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 751             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 752             if (extraRoute != null) {
 753                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 754                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 755                     if (nextHopIp != null) {
 756                         localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
 757                         localNextHopIP = nextHopIp + "/32";
 758                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 759                         returnLocalDpnId.add(dpnId);
 760                     }
 761                 }
 762             }
 763             if (localNextHopInfo == null) {
 764             /* imported routes case */
 765                 synchronized (vrfEntry.getLabel().toString().intern()) {
 766                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 767                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 768                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 769                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 770                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 771                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 772                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 773                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 774                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 775                                     localNextHopIP = lri.getPrefix();
 776                                 } else {
 777                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 778                                     localNextHopIP = lri.getPrefix();
 779                                 }
 780                             }
 781                             if (localNextHopInfo != null) {
 782                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 783                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 784                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 785                                 returnLocalDpnId.add(dpnId);
 786                             }
 787                         }
 788                     }
 789                 }
 790             }
 791         } else {
 792             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 793             returnLocalDpnId.add(dpnId);
 794         }
 795
 796         return returnLocalDpnId;
 797     }
 798
 799     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 800                                                 final VrfEntry vrfEntry, Long parentVpnId){
 801         if (localNextHopInfo != null) {
 802             final BigInteger dpnId = localNextHopInfo.getDpnId();
 803             if (!isVpnPresentInDpn(rd, dpnId)) {
 804                 return BigInteger.ZERO;
 805             }
 806
 807             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
 808
 809             List<ActionInfo> actionsInfos =
 810                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 811             final List<InstructionInfo> instructions =
 812                     Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 813             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 814                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 815             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 816             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 817                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 818                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 819             } else {
 820                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 821             }
 822             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 823             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 824                     new Callable<List<ListenableFuture<Void>>>() {
 825                         @Override
 826                         public List<ListenableFuture<Void>> call() throws Exception {
 827                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 828                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 829                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 830                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 831                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 832                             }
 833                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 834                             futures.add(tx.submit());
 835                             return futures;
 836                         }
 837                     });
 838             return dpnId;
 839         }
 840         return BigInteger.ZERO;
 841     }
 842
 843     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 844         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 845         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 846         if (dpnInVpn.isPresent()) {
 847             return true;
 848         }
 849         return false;
 850     }
 851
 852     private LabelRouteInfo getLabelRouteInfo(Long label) {
 853         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 854                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 855         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 856         if (opResult.isPresent()) {
 857             return opResult.get();
 858         }
 859         return null;
 860     }
 861
 862     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 863         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 864         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 865                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 866         if (lri == null) {
 867             return true;
 868         }
 869         List<String> vpnInstancesList = lri.getVpnInstanceList();
 870         if (vpnInstancesList.contains(vpnInstanceName)) {
 871             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 872             vpnInstancesList.remove(vpnInstanceName);
 873         }
 874         if (vpnInstancesList.size() == 0) {
 875             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 876             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 877             return true;
 878         } else {
 879             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 880             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 881             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 882         }
 883         return false;
 884     }
 885
 886     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 887                                       WriteTransaction tx) {
 888         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 889         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 890
 891
 892         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 893
 894         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 895                 dpId, label, groupId);
 896     }
 897
 898     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 899                                                  WriteTransaction tx) {
 900         List<MatchInfo> mkMatches = new ArrayList<>();
 901
 902         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 903
 904         // Matching metadata
 905         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 906         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 907
 908         List<InstructionInfo> mkInstructions = new ArrayList<>();
 909         mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 910
 911         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 912                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 913                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 914
 915         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 916
 917         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 918
 919         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 920         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 921                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 922                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 923         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 924     }
 925
 926     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 927         FlowEntity flowEntity;
 928         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 929         List<MatchInfo> mkMatches = new ArrayList<>();
 930         // Matching metadata
 931         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 932         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 933                 NwConstants.INTERNAL_TUNNEL_TABLE,
 934                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 935                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 936                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 937         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 938         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 939         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 940                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 941                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 942
 943         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 944         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 945     }
 946
 947     /**
 948      * Delete local FIB entry
 949      * @param vpnId
 950      * @param rd
 951      * @param vrfEntry
 952      * @return
 953      */
 954     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 955         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 956         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 957         String localNextHopIP = vrfEntry.getDestPrefix();
 958
 959         if (localNextHopInfo == null) {
 960             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 961             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 962             if (extra_route != null) {
 963                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 964                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 965                     if (nextHopIp != null) {
 966                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 967                         localNextHopIP = nextHopIp + "/32";
 968                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 969                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 970                         if (!dpnId.equals(BigInteger.ZERO)) {
 971                             returnLocalDpnId.add(dpnId);
 972                         }
 973                     }
 974                 }
 975             }
 976
 977             if (localNextHopInfo == null) {
 978               /* Imported VRF entry */
 979                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 980                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 981                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 982                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
 983                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
 984                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
 985                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 986                     if (!dpnId.equals(BigInteger.ZERO)) {
 987                         returnLocalDpnId.add(dpnId);
 988                     }
 989                 }
 990             }
 991
 992
 993         } else {
 994             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 995                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 996             if (!dpnId.equals(BigInteger.ZERO)) {
 997                 returnLocalDpnId.add(dpnId);
 998             }
 999         }
1000
1001         return returnLocalDpnId;
1002     }
1003
1004     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1005                                                 final Long vpnId, final String rd,
1006                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1007         if (localNextHopInfo != null) {
1008             final BigInteger dpnId = localNextHopInfo.getDpnId();;
1009             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1010             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
1011                     new Callable<List<ListenableFuture<Void>>>() {
1012                         @Override
1013                         public List<ListenableFuture<Void>> call() throws Exception {
1014                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1015                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1016                                     NwConstants.DEL_FLOW, tx);
1017                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1018                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1019                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1020                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1021                             }
1022                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1023                             futures.add(tx.submit());
1024                             return futures;
1025                         }
1026                     });
1027             //TODO: verify below adjacency call need to be optimized (?)
1028             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
1029             return dpnId;
1030         }
1031         return BigInteger.ZERO;
1032     }
1033
1034
1035     private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
1036         return InstanceIdentifier.builder(PrefixToInterface.class)
1037                 .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
1038     }
1039
1040     private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
1041         Optional<Prefixes> localNextHopInfoData =
1042                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
1043         return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
1044     }
1045
1046     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1047         return InstanceIdentifier.builder(VpnToExtraroute.class)
1048                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1049                         new ExtrarouteKey(ipPrefix)).build();
1050     }
1051
1052     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1053         Optional<Extraroute> extraRouteInfo =
1054                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1055         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1056
1057     }
1058
1059     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1060         try {
1061             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1062                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1063             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1064             if(!rpcResult.isSuccessful()) {
1065                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1066             } else {
1067                 return rpcResult.getResult().getTunnelType();
1068             }
1069
1070         } catch (InterruptedException | ExecutionException e) {
1071             LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
1072         }
1073
1074         return null;
1075
1076     }
1077     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1078                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1079         Boolean wrTxPresent = true;
1080         if (tx == null) {
1081             wrTxPresent = false;
1082             tx = dataBroker.newWriteOnlyTransaction();
1083         }
1084         String rd = vrfTableKey.getRouteDistinguisher();
1085         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1086                 vrfEntry.getDestPrefix(), rd, tx);
1087         /********************************************/
1088         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1089
1090         if (tunnelInterfaceList.isEmpty()) {
1091             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1092                     vrfEntry.getNextHopAddressList(), rd);
1093             LOG.warn("Failed to add Route: {} in vpn: {}",
1094                     vrfEntry.getDestPrefix(), rd);
1095             return;
1096         }
1097
1098         for (String tunnelInterface : tunnelInterfaceList) {
1099             List<InstructionInfo> instructions = new ArrayList<>();
1100             List<ActionInfo> actionInfos = new ArrayList<>();
1101             Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1102             if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1103                 LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1104                 actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
1105                 actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
1106             } else {
1107                 int label = vrfEntry.getLabel().intValue();
1108                 BigInteger tunnelId;
1109                 // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1110                 if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1111                     tunnelId = BigInteger.valueOf(label);
1112                 } else {
1113                     tunnelId = BigInteger.valueOf(label);
1114                 }
1115
1116                 LOG.debug("adding set tunnel id action for label {}", label);
1117                 actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
1118             }
1119             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
1120             if(egressActions.isEmpty()){
1121                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
1122                 return;
1123             }
1124             actionInfos.addAll(egressActions);
1125             instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
1126             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1127         }
1128         if(!wrTxPresent ){
1129             tx.submit();
1130         }
1131         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1132     }
1133
1134     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1135         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1136         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1137         if (dpnInVpn.isPresent()) {
1138             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1139             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1140
1141             if (vpnInterfaces.remove(currVpnInterface)) {
1142                 if (vpnInterfaces.isEmpty()) {
1143                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1144                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1145                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1146                 } else {
1147                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1148                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1149                             VpnInterfaces.class,
1150                             new VpnInterfacesKey(intfName)));
1151                 }
1152             }
1153         }
1154     }
1155
1156     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1157     /* Get interface info from prefix to interface mapping;
1158         Use the interface info to get the corresponding vpn interface op DS entry,
1159         remove the adjacency corresponding to this fib entry.
1160         If adjacency removed is the last adjacency, clean up the following:
1161          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1162          - prefix to interface entry
1163          - vpn interface op DS
1164      */
1165         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1166         Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1167         Extraroute extraRoute = null;
1168         if (prefixInfo == null) {
1169             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1170             if(extraRoute != null) {
1171                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1172                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1173
1174                     if (nextHopIp != null) {
1175                         prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
1176                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1177                     }
1178                 }
1179             }
1180             if (prefixInfo == null) {
1181                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1182                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1183                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1184                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1185                     prefixBuilder.setDpnId(lri.getDpnId());
1186                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1187                     prefixBuilder.setIpAddress(lri.getPrefix());
1188                     prefixInfo = prefixBuilder.build();
1189                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1190                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1191                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1192                 }
1193             }
1194         } else {
1195             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1196         }
1197     }
1198
1199     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1200                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1201
1202         if (prefixInfo == null) {
1203             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1204             return; //Don't have any info for this prefix (shouldn't happen); need to return
1205         }
1206
1207         String ifName = prefixInfo.getVpnInterfaceName();
1208         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1209         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1210                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1211     }
1212
1213     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1214         Prefixes prefixInfo;
1215         Long vpnId;
1216         String rd;
1217         VrfEntry vrfEntry;
1218         Extraroute extraRoute;
1219
1220         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1221                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1222             this.prefixInfo = prefixInfo;
1223             this.vpnId = vpnId;
1224             this.rd= rd;
1225             this.vrfEntry= vrfEntry;
1226             this.extraRoute = extraRoute;
1227         }
1228
1229         @Override
1230         public List<ListenableFuture<Void>> call() throws Exception {
1231             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1232             // to call the respective helpers.
1233             String ifName = prefixInfo.getVpnInterfaceName();
1234             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1235                     FibUtil.getVpnInterfaceIdentifier(ifName));
1236             if (optvpnInterface.isPresent()) {
1237                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1238                 if (vpnId != associatedVpnId) {
1239                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1240                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1241                     LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
1242                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1243                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1244                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1245                     return null;
1246                 } else {
1247                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1248                             vrfEntry.getDestPrefix(), associatedVpnId);
1249                 }
1250             }
1251             if (extraRoute != null) {
1252                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1253                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1254             }
1255             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1256                     FibUtil.getAdjListPath(ifName));
1257             int numAdj = 0;
1258             if (optAdjacencies.isPresent()) {
1259                 numAdj = optAdjacencies.get().getAdjacency().size();
1260             }
1261             //remove adjacency corr to prefix
1262             if (numAdj > 1) {
1263                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1264                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1265                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1266             }
1267             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1268                 //clean up the vpn interface from DpnToVpn list
1269                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1270                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1271                         FibUtil.getVpnInterfaceIdentifier(ifName));
1272             }
1273
1274             synchronized (vrfEntry.getLabel().toString().intern()) {
1275                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1276                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1277                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1278                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1279                     String vpnInstanceName = "";
1280                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1281                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1282                     }
1283                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1284                     if (lriRemoved) {
1285                         String parentRd = lri.getParentVpnRd();
1286                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1287                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1288                     }
1289                 } else {
1290                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1291                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1292                 }
1293             }
1294             return null;
1295         }
1296     }
1297
1298     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1299         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1300         final String rd  = vrfTableKey.getRouteDistinguisher();
1301         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1302         if (vpnInstance == null) {
1303             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1304             return;
1305         }
1306         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1307         long elanTag = 0L;
1308         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1309         if (subnetRoute != null) {
1310             elanTag = subnetRoute.getElantag();
1311             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1312                     rd, vrfEntry.getDestPrefix(), elanTag);
1313             if (vpnToDpnList != null) {
1314                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1315                 dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1316                         new Callable<List<ListenableFuture<Void>>>() {
1317                             @Override
1318                             public List<ListenableFuture<Void>> call() throws Exception {
1319                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1320
1321                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1322
1323                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1324                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1325                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1326                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1327                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1328                                     }
1329                                 }
1330                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1331                                 futures.add(tx.submit());
1332                                 return futures;
1333                             }
1334                         });
1335             }
1336             synchronized (vrfEntry.getLabel().toString().intern()) {
1337                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1338                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1339                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1340                     String vpnInstanceName = "";
1341                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1342                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1343                     }
1344                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1345                     if (lriRemoved) {
1346                         String parentRd = lri.getParentVpnRd();
1347                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1348                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1349                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1350                                 vrfEntry.getDestPrefix());
1351                     }
1352                 } else {
1353                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1354                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1355                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1356                             vrfEntry.getDestPrefix());
1357                 }
1358             }
1359             return;
1360         }
1361
1362         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1363                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1364         if (vpnToDpnList != null) {
1365             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1366             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1367                     new Callable<List<ListenableFuture<Void>>>() {
1368                         @Override
1369                         public List<ListenableFuture<Void>> call() throws Exception {
1370                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1371
1372                             if (localDpnIdList.size() <= 0) {
1373                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1374                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1375                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1376                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1377                                         }
1378                                     } else {
1379                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1380                                     }
1381                                 }
1382                             } else {
1383                                 for (BigInteger localDpnId : localDpnIdList) {
1384                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1385                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1386                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1387                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1388                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1389                                                 }
1390                                             } else {
1391                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1392                                             }
1393                                         }
1394                                     }
1395                                 }
1396                             }
1397                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1398                             futures.add(tx.submit());
1399                             return futures;
1400                         }
1401                     });
1402         }
1403
1404         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1405         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1406         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1407
1408         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1409         // of the interVpnLink.
1410         Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
1411         if ( vpnUuid.isPresent() ) {
1412             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, vpnUuid.get());
1413             String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
1414
1415             if ( interVpnLink.isPresent()
1416                     && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
1417                     && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
1418                     || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
1419                     && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
1420                 // This is route that points to the other endpoint of an InterVpnLink
1421                 // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1422                 removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
1423             }
1424         }
1425
1426     }
1427
1428     /*
1429       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1430       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1431       provided by ResourceBatchingManager
1432      */
1433     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1434         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1435
1436         final String rd  = vrfTableKey.getRouteDistinguisher();
1437         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1438         if (vpnInstance == null) {
1439             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1440             return;
1441         }
1442         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1443         if (vpnToDpnList != null) {
1444             for (VpnToDpnList curDpn : vpnToDpnList) {
1445                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1446                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1447                 }
1448             }
1449         }
1450     }
1451
1452     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1453                                   final long vpnId, final VrfTablesKey vrfTableKey,
1454                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1455
1456         Boolean wrTxPresent = true;
1457         if (tx == null) {
1458             wrTxPresent = false;
1459             tx = dataBroker.newWriteOnlyTransaction();
1460         }
1461
1462         LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1463         String rd = vrfTableKey.getRouteDistinguisher();
1464
1465         if(localDpnId != null) {
1466             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1467             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1468             return;
1469         }
1470
1471         // below two reads are kept as is, until best way is found to identify dpnID
1472         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1473         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1474
1475         if (localNextHopInfo == null && extraRoute != null) {
1476             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1477             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1478                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1479                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1480             }
1481         } else {
1482             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1483         }
1484         if(!wrTxPresent ){
1485             tx.submit();
1486         }
1487     }
1488
1489     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1490                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1491         boolean isRemoteRoute = true;
1492         if (localNextHopInfo != null) {
1493             isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId()));
1494         }
1495         if (isRemoteRoute) {
1496             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1497             return true;
1498         } else {
1499             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1500             return false;
1501         }
1502     }
1503
1504     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1505         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1506         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1507     }
1508
1509     private long get
1510             (byte[] rawIpAddress) {
1511         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1512                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1513     }
1514
1515     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1516                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1517         Boolean wrTxPresent = true;
1518         if (tx == null) {
1519             wrTxPresent = false;
1520             tx = dataBroker.newWriteOnlyTransaction();
1521         }
1522
1523         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1524         String values[] = vrfEntry.getDestPrefix().split("/");
1525         String ipAddress = values[0];
1526         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1527         if (addOrRemove == NwConstants.ADD_FLOW) {
1528             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1529         } else {
1530             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1531         }
1532         InetAddress destPrefix;
1533         try {
1534             destPrefix = InetAddress.getByName(ipAddress);
1535         } catch (UnknownHostException e) {
1536             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1537             return;
1538         }
1539
1540         List<MatchInfo> matches = new ArrayList<>();
1541
1542         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1543                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1544
1545         matches.add(new MatchInfo(MatchFieldType.eth_type,
1546                 new long[] { NwConstants.ETHTYPE_IPV4 }));
1547
1548         if(prefixLength != 0) {
1549             matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1550                     destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1551         }
1552         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1553         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1554         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1555                 COOKIE_VM_FIB_TABLE, matches, instructions);
1556
1557         Flow flow = flowEntity.getFlowBuilder().build();
1558         String flowId = flowEntity.getFlowId();
1559         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1560         Node nodeDpn = buildDpnNode(dpId);
1561
1562         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1563                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1564                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1565         if (addOrRemove == NwConstants.ADD_FLOW) {
1566             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1567         } else {
1568             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1569         }
1570
1571         if(!wrTxPresent ){
1572             tx.submit();
1573         }
1574     }
1575
1576     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1577     private Node buildDpnNode(BigInteger dpnId) {
1578         NodeId nodeId = new NodeId("openflow:" + dpnId);
1579         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1580
1581         return nodeDpn;
1582     }
1583
1584     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1585                                     int addOrRemove, WriteTransaction tx) {
1586         Boolean wrTxPresent = true;
1587         if (tx == null) {
1588             wrTxPresent = false;
1589             tx = dataBroker.newWriteOnlyTransaction();
1590         }
1591
1592         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1593         matches.add(new MatchInfo(MatchFieldType.eth_type,
1594                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1595         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1596
1597         // Install the flow entry in L3_LFIB_TABLE
1598         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1599
1600         FlowEntity flowEntity;
1601         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1602                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1603         Flow flow = flowEntity.getFlowBuilder().build();
1604         String flowId = flowEntity.getFlowId();
1605         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1606         Node nodeDpn = buildDpnNode(dpId);
1607         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1608                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1609                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1610
1611         if (addOrRemove == NwConstants.ADD_FLOW) {
1612             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1613         } else {
1614             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1615         }
1616         if(!wrTxPresent ){
1617             tx.submit();
1618         }
1619         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
1620                 dpId, label, instructions );
1621     }
1622
1623     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
1624         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1625         try {
1626             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
1627         } catch (NullPointerException e) {
1628             LOG.trace("", e);
1629         }
1630     }
1631
1632     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1633                                     final FutureCallback<List<Void>> callback) {
1634         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1635         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1636         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1637         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1638         if (vrfTable.isPresent()) {
1639             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1640             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1641                     new Callable<List<ListenableFuture<Void>>>() {
1642                         @Override
1643                         public List<ListenableFuture<Void>> call() throws Exception {
1644                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1645                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1646                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1647                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1648
1649                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1650                                     if (subnetRoute != null) {
1651                                         long elanTag = subnetRoute.getElantag();
1652                                         installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1653                                         continue;
1654                                     }
1655                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1656                                         //Handle local flow creation for imports
1657                                         LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1658                                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()
1659                                         ) && vrfEntry.getNextHopAddressList()
1660                                                 .contains(lri.getNextHopIpList().get(0))) {
1661                                             if (lri.getDpnId().equals(dpnId)) {
1662                                                 createLocalFibEntry(vpnId, rd, vrfEntry);
1663                                                 continue;
1664                                             }
1665                                         }
1666                                     }
1667                                     // Passing null as we don't know the dpn
1668                                     // to which prefix is attached at this point
1669                                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1670                                 }
1671                                 //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1672                                 futures.add(tx.submit());
1673                                 if (callback != null) {
1674                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1675                                     Futures.addCallback(listenableFuture, callback);
1676                                 }
1677                             }
1678                             return futures;
1679                         }
1680                     });
1681         }
1682     }
1683
1684     public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1685                                  final String localNextHopIp, final String remoteNextHopIp) {
1686         LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1687                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1688         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1689         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1690         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1691         if (vrfTable.isPresent()) {
1692             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1693             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1694                     new Callable<List<ListenableFuture<Void>>>() {
1695                         @Override
1696                         public List<ListenableFuture<Void>> call() throws Exception {
1697                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1698                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1699                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1700                                 LOG.trace("populate FIB starts on Dpn " + dpnId
1701                                         + "rd  " + rd.toString()
1702                                         + "localNextHopIp " + localNextHopIp
1703                                         + "remoteNextHopIp" + remoteNextHopIp
1704                                         + "vpnId " + vpnId );
1705                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1706                                     LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1707
1708                                     if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1709                                         if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1710                                             LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1711                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1712                                         }
1713                                     } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
1714                                         BigInteger dpnIdForPrefix = null;
1715                                         String destPfx = vrfEntry.getDestPrefix();
1716                                         if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
1717                                             Optional<Extraroute> extraRouteInfo =
1718                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1719                                                             getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1720                                             if (extraRouteInfo.isPresent()) {
1721                                                 continue;
1722                                             }
1723                                             dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1724                                         } else {
1725                                             // Subnet Route handling
1726                                             Optional<Prefixes> localNextHopInfoData =
1727                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1728                                                             FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
1729                                             if (localNextHopInfoData.isPresent()) {
1730                                                 Prefixes prefixes = localNextHopInfoData.get();
1731                                                 dpnIdForPrefix = prefixes.getDpnId();
1732                                             }
1733                                         }
1734                                         if (dpnIdForPrefix == null) {
1735                                             LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1736                                                     vrfEntry.getDestPrefix());
1737                                             continue;
1738                                         }
1739                                         int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1740                                         if (sameDpnId != 0) {
1741                                             LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1742                                                     dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1743                                             continue;
1744                                         }
1745                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1746                                         List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
1747                                         newNextHopAddrList.add(localNextHopIp);
1748                                         VrfEntry newVrfEntry =
1749                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
1750                                         // Just update the VrfEntry
1751                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1752                                                 vrfEntryId, newVrfEntry);
1753                                         // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
1754                                         //       vrfEntryId, newVrfEntry);
1755                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1756                                         LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1757                                     }
1758                                 }
1759                                 futures.add(writeTransaction.submit());
1760                                 LOG.trace("populate FIB ends on Dpn " + dpnId
1761                                         + "rd  " + rd.toString()
1762                                         + "localNextHopIp " + localNextHopIp
1763                                         + "remoteNextHopIp" + remoteNextHopIp
1764                                         + "vpnId " + vpnId);
1765                             }
1766                             return futures;
1767                         }
1768                     });
1769         }
1770     }
1771
1772     public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
1773                                   final long vpnId, final String  rd, final String destPrefix ,
1774                                   final String localNextHopIP, final String remoteNextHopIp) {
1775
1776         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1777         dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1778                         + "local dpid" + localDpnId
1779                         + "remote dpid" + remoteDpnId
1780                         + "vpnId" + vpnId
1781                         + "localNHIp" + localNextHopIP
1782                         + "remoteNHIp" + remoteNextHopIp,
1783                 new Callable<List<ListenableFuture<Void>>>() {
1784                     @Override
1785                     public List<ListenableFuture<Void>> call() throws Exception {
1786                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1787                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1788                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1789                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1790                         if (vrfEntry == null)
1791                             return futures;
1792                         LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1793                                         "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1794                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1795                         if (action == true) {
1796                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1797                             LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1798                             createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1799                         } else {
1800                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1801                             LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1802                             deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1803                         }
1804                         futures.add(writeTransaction.submit());
1805                         return futures;
1806                     }
1807                 });
1808     }
1809
1810     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1811                                  final FutureCallback<List<Void>> callback) {
1812         LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1813         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1814         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1815         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1816         if (vrfTable.isPresent()) {
1817             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1818             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1819                     new Callable<List<ListenableFuture<Void>>>() {
1820                         @Override
1821                         public List<ListenableFuture<Void>> call() throws Exception {
1822                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1823                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1824                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1825                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1826                                     /* Handle subnet routes here */
1827                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1828                                     if (subnetRoute != null) {
1829                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1830                                                 dpnId, rd);
1831                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1832                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1833                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1834                                                 vrfEntry.getDestPrefix());
1835                                         continue;
1836                                     }
1837                                     // Passing null as we don't know the dpn
1838                                     // to which prefix is attached at this point
1839                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1840                                 }
1841                                 futures.add(tx.submit());
1842                                 if (callback != null) {
1843                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1844                                     Futures.addCallback(listenableFuture, callback);
1845                                 }
1846                             }
1847                             return futures;
1848                         }
1849
1850                     });
1851         }
1852     }
1853
1854     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1855                                  final String localNextHopIp, final String remoteNextHopIp,
1856                                  final FutureCallback<List<Void>> callback) {
1857         LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1858                         " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1859                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1860         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1861         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1862         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1863         if (vrfTable.isPresent()) {
1864             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1865             dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
1866                     new Callable<List<ListenableFuture<Void>>>() {
1867                         @Override
1868                         public List<ListenableFuture<Void>> call() throws Exception {
1869                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1870                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1871                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1872                                 LOG.trace("cleanup FIB starts on Dpn " + dpnId
1873                                         + "rd  " + rd.toString()
1874                                         + "localNextHopIp " + localNextHopIp
1875                                         + "remoteNextHopIp" + remoteNextHopIp
1876                                         + "vpnId " + vpnId);
1877
1878                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1879                                     LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1880                                     if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1881                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1882                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1883                                     }
1884
1885                                     if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1886                                         LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1887                                                 vrfEntry.getDestPrefix(), dpnId);
1888                                         String destPfx = vrfEntry.getDestPrefix();
1889                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1890                                         List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1891                                         newList.remove(localNextHopIp);
1892                                         VrfEntry newVrfEntry =
1893                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1894                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1895                                                 vrfEntryId, newVrfEntry);
1896                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1897                                         LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1898                                     }
1899                                 }
1900                                 futures.add(writeTransaction.submit());
1901                                 if (callback != null) {
1902                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1903                                     Futures.addCallback(listenableFuture, callback);
1904                                 }
1905                                 LOG.trace("cleanup FIB ends on Dpn " + dpnId
1906                                         + "rd  " + rd.toString()
1907                                         + "localNextHopIp " + localNextHopIp
1908                                         + "remoteNextHopIp" + remoteNextHopIp
1909                                         + "vpnId " + vpnId);
1910                             }
1911                             return futures;
1912                         }
1913                     });
1914
1915         }
1916     }
1917
1918     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1919         InstanceIdentifierBuilder<VrfTables> idBuilder =
1920                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1921         InstanceIdentifier<VrfTables> id = idBuilder.build();
1922         return id;
1923     }
1924
1925     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1926         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1927                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1928                 .append(priority).toString();
1929     }
1930
1931     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1932         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1933                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1934                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1935                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1936                 .append(destPrefix.getHostAddress()).toString();
1937     }
1938
1939     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1940         return new StringBuilder(64).append(FLOWID_PREFIX)
1941                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1942                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1943                 .append(nextHop).toString();
1944     }
1945
1946     protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
1947                                             final VrfEntry vrfEntry, String rd) {
1948         List<String> adjacencyList = new ArrayList<>();
1949         List<String> prefixIpList = new ArrayList<>();
1950         LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
1951                 remoteDpnId, vpnId, vrfEntry);
1952         try {
1953             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1954                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1955                 if (extra_route == null) {
1956                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1957                 } else {
1958                     prefixIpList = new ArrayList<>();
1959                     for (String extraRouteIp : extra_route.getNexthopIpList()) {
1960                         prefixIpList.add(extraRouteIp + "/32");
1961                     }
1962                 }
1963             } else {
1964                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1965             }
1966
1967             for (String prefixIp : prefixIpList) {
1968                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
1969                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
1970                     String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
1971                             prefixIp, nextHopIp);
1972                     if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
1973                         adjacencyList.add(adjacency);
1974                     }
1975                 }
1976             }
1977         } catch (NullPointerException e) {
1978             LOG.trace("", e);
1979         }
1980         return adjacencyList;
1981     }
1982
1983     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
1984         InstanceIdentifier<VpnInstanceOpDataEntry> id =
1985                 InstanceIdentifier.create(VpnInstanceOpData.class)
1986                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
1987         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
1988                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1989         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
1990     }
1991
1992     public void processNodeAdd(BigInteger dpnId) {
1993         LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId);
1994         makeTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1995         makeL3IntfTblMissFlow(dpnId, NwConstants.ADD_FLOW);
1996         makeSubnetRouteTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1997     }
1998
1999     private void makeTableMissFlow(BigInteger dpnId, int addOrRemove) {
2000         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2001         // Instruction to goto L3 InterfaceTable
2002         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2003         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE }));
2004         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2005         FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_LFIB_TABLE,
2006                 getTableMissFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW),
2007                 NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
2008
2009         FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE,
2010                 getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE,
2011                         NwConstants.TABLE_MISS_FLOW),
2012                 NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow",
2013                 0, 0, COOKIE_VM_FIB_TABLE,
2014                 matches, instructions);
2015
2016         if (addOrRemove == NwConstants.ADD_FLOW) {
2017             LOG.debug("Invoking MDSAL to install Table Miss Entries");
2018             mdsalManager.installFlow(flowEntityLfib);
2019             mdsalManager.installFlow(flowEntityFib);
2020         } else {
2021             mdsalManager.removeFlow(flowEntityLfib);
2022             mdsalManager.removeFlow(flowEntityFib);
2023
2024         }
2025     }
2026
2027     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2028         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2029                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2030                 .append(FLOWID_PREFIX).toString();
2031     }
2032
2033     /*
2034      * Install flow entry in protocol table to forward mpls
2035      * coming through gre tunnel to LFIB table.
2036      */
2037     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2038         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2039         // Instruction to goto L3 InterfaceTable
2040         List<InstructionInfo> instructions = new ArrayList<>();
2041         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2042         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2043         matches.add(new MatchInfo(MatchFieldType.eth_type,
2044                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2045         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2046                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2047                         NwConstants.L3_LFIB_TABLE),
2048                 DEFAULT_FIB_FLOW_PRIORITY,
2049                 "Protocol Table For LFIB",
2050                 0, 0,
2051                 COOKIE_PROTOCOL_TABLE,
2052                 matches, instructions);
2053
2054         if (addOrRemove == NwConstants.ADD_FLOW) {
2055             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2056             mdsalManager.installFlow(flowEntityToLfib);
2057         } else {
2058             mdsalManager.removeFlow(flowEntityToLfib);
2059         }
2060     }
2061
2062     public List<String> printFibEntries() {
2063         List<String> result = new ArrayList<String>();
2064         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2065         result.add("-------------------------------------------------------------------");
2066         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2067         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2068         if (fibEntries.isPresent()) {
2069             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2070             for (VrfTables vrfTable : vrfTables) {
2071                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2072                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2073                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2074                                 vrfTable.getRouteDistinguisher(),
2075                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2076                     }
2077                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2078                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2079                                 vrfTable.getRouteDistinguisher(),
2080                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2081                     }
2082                 }
2083             }
2084         }
2085         return result;
2086     }
2087
2088     private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) {
2089         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2090         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2091         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2092         // Instruction to goto L3 InterfaceTable
2093
2094         List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
2095         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
2096                 Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
2097         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2098         //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
2099
2100         FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
2101                 getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
2102                 NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS,
2103                 matches, instructions);
2104         if (addOrRemove == NwConstants.ADD_FLOW) {
2105             LOG.debug("Invoking MDSAL to install L3 interface Table Miss Entries");
2106             mdsalManager.installFlow(flowEntityL3Intf);
2107         } else {
2108             mdsalManager.removeFlow(flowEntityL3Intf);
2109         }
2110     }
2111
2112     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2113         InstanceIdentifier<VrfEntry> vrfEntryId =
2114                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2115                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2116         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2117         if (vrfEntry.isPresent())  {
2118             return (vrfEntry.get());
2119         }
2120         return null;
2121     }
2122
2123     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2124         InstanceIdentifier<VrfEntry> vrfEntryId =
2125                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2126                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2127         return vrfEntryId;
2128     }
2129 }