vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15
  16 import java.math.BigInteger;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import java.util.concurrent.LinkedBlockingQueue;
  28
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  31 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  34 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.ActionType;
  37 import org.opendaylight.genius.mdsalutil.FlowEntity;
  38 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  39 import org.opendaylight.genius.mdsalutil.InstructionType;
  40 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  41 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  42 import org.opendaylight.genius.mdsalutil.MatchInfo;
  43 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.packet.IPProtocols;
  47 import org.opendaylight.genius.utils.ServiceIndex;
  48 import org.opendaylight.genius.utils.batching.ActionableResource;
  49 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  50 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  51 import org.opendaylight.genius.utils.batching.ResourceHandler;
  52 import org.opendaylight.genius.utils.batching.SubTransaction;
  53 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  54 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  55 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  56 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  57 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  58 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  59 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  60 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 120 import org.opendaylight.yangtools.yang.binding.DataObject;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 123 import org.opendaylight.yangtools.yang.common.RpcResult;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> implements AutoCloseable, ResourceHandler {
 128     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 129     private static final String FLOWID_PREFIX = "L3.";
 130     private final DataBroker dataBroker;
 131     private final IMdsalApiManager mdsalManager;
 132     private IVpnManager vpnmanager;
 133     private final NexthopManager nextHopManager;
 134     private ItmRpcService itmManager;
 135     private final OdlInterfaceRpcService interfaceManager;
 136     private final IdManagerService idManager;
 137     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 138     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 139     private static final int LFIB_INTERVPN_PRIORITY = 1;
 140     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 141     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 142     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 143     List<SubTransaction> transactionObjects;
 144     private static final int PERIODICITY = 500;
 145     private static Integer batchSize;
 146     private static Integer batchInterval;
 147     private static final int BATCH_SIZE = 1000;
 148     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 149     private final ResourceBatchingManager resourceBatchingManager;
 150
 151     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 152                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 153                             final IdManagerService idManager) {
 154         super(VrfEntry.class, VrfEntryListener.class);
 155         this.dataBroker = dataBroker;
 156         this.mdsalManager = mdsalApiManager;
 157         this.nextHopManager = nexthopManager;
 158         this.interfaceManager = interfaceManager;
 159         this.idManager = idManager;
 160
 161         batchSize = Integer.getInteger("batch.size");
 162         if (batchSize == null) {
 163             batchSize = BATCH_SIZE;
 164         }
 165         batchInterval = Integer.getInteger("batch.wait.time");
 166         if (batchInterval == null) {
 167             batchInterval = PERIODICITY;
 168         }
 169         resourceBatchingManager = ResourceBatchingManager.getInstance();
 170         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 171         transactionObjects = new ArrayList<>();
 172     }
 173
 174     public void start() {
 175         LOG.info("{} start", getClass().getSimpleName());
 176         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 177     }
 178
 179     @Override
 180     protected VrfEntryListener getDataTreeChangeListener() { return VrfEntryListener.this; }
 181
 182     @Override
 183     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 184         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 185     }
 186
 187     @Override
 188     public DataBroker getResourceBroker() {
 189         return dataBroker;
 190     }
 191
 192     @Override
 193     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 194         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 195         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 196         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 197                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 198         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 199             createFibEntries(identifier, vrfEntry);
 200         } else {
 201             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 202             actResource.setAction(ActionableResource.CREATE);
 203             actResource.setInstanceIdentifier(identifier);
 204             actResource.setInstance(vrfEntry);
 205             vrfEntryBufferQ.add(actResource);
 206             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 207         }
 208         LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 209                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 210     }
 211
 212     @Override
 213     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 214         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 215         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 216         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 217                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 218         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 219             deleteFibEntries(identifier, vrfEntry);
 220         } else {
 221             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 222             actResource.setAction(ActionableResource.DELETE);
 223             actResource.setInstanceIdentifier(identifier);
 224             actResource.setInstance(vrfEntry);
 225             vrfEntryBufferQ.add(actResource);
 226             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 227         }
 228         LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 229                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 230     }
 231
 232     @Override
 233     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 234         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 235         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 236         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 237                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 238         if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 239             createFibEntries(identifier, update);
 240         } else {
 241             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 242             actResource.setAction(ActionableResource.UPDATE);
 243             actResource.setInstanceIdentifier(identifier);
 244             actResource.setInstance(update);
 245             actResource.setOldInstance(original);
 246             vrfEntryBufferQ.add(actResource);
 247         }
 248         LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 249                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 250     }
 251
 252     @Override
 253     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
 254         this.transactionObjects = transactionObjects;
 255         if (vrfEntry instanceof VrfEntry) {
 256             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 257         }
 258     }
 259
 260     @Override
 261     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
 262         this.transactionObjects = transactionObjects;
 263         if (vrfEntry instanceof VrfEntry) {
 264             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 265         }
 266     }
 267
 268     @Override
 269     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 270                        Object update, List<SubTransaction> transactionObjects) {
 271         this.transactionObjects = transactionObjects;
 272         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 273             createFibEntries(tx, identifier, (VrfEntry)update);
 274         }
 275     }
 276
 277     @Override
 278     public int getBatchSize() {
 279         return batchSize;
 280     }
 281
 282     @Override
 283     public int getBatchInterval() {
 284         return batchInterval;
 285     }
 286
 287     @Override
 288     public LogicalDatastoreType getDatastoreType() {
 289         return LogicalDatastoreType.CONFIGURATION;
 290     }
 291
 292     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 293         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 294
 295         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 296         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 297         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 298
 299         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 300         final Long vpnId = vpnInstance.getVpnId();
 301         final String rd = vrfTableKey.getRouteDistinguisher();
 302         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 303         if (subnetRoute != null) {
 304             final long elanTag = subnetRoute.getElantag();
 305             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 306                     rd, vrfEntry.getDestPrefix(), elanTag);
 307             if (vpnToDpnList != null) {
 308                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 309                 dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 310                         new Callable<List<ListenableFuture<Void>>>() {
 311                             @Override
 312                             public List<ListenableFuture<Void>> call() throws Exception {
 313                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 314                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 315                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 316                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 317                                     }
 318                                 }
 319                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 320                                 futures.add(tx.submit());
 321                                 return futures;
 322                             }
 323                         });
 324             }
 325             return;
 326         }
 327         // ping responder for router interfaces
 328         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 329             return;
 330         }
 331
 332         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 333             // When it is a leaked route, the LFIB and FIB goes a bit different.
 334             installInterVpnRouteInLFib(rd, vrfEntry);
 335             return;
 336         }
 337
 338         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 339
 340         if (vpnToDpnList != null) {
 341             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 342             dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 343                     new Callable<List<ListenableFuture<Void>>>() {
 344                         @Override
 345                         public List<ListenableFuture<Void>> call() throws Exception {
 346                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 347                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 348                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 349                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 350                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 351                                     }
 352                                 }
 353                             }
 354                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 355                             futures.add(tx.submit());
 356                             return futures;
 357                         }
 358                     });
 359         }
 360
 361         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 362         if ( optVpnUuid.isPresent() ) {
 363             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 364             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 365             if ( optInterVpnLink.isPresent() ) {
 366                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 367                 String vpnUuid = optVpnUuid.get();
 368                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 369                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid) ) {
 370                     // This is an static route that points to the other endpoint of an InterVpnLink
 371                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 372                     installRouteInInterVpnLink(interVpnLink, vpnUuid, vrfEntry, vpnId);
 373                     installInterVpnRouteInLFib(rd, vrfEntry);
 374                 }
 375             }
 376         }
 377     }
 378
 379
 380     /*
 381       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 382       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 383       provided by ResourceBatchingManager
 384      */
 385     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 386         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 387
 388         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 389         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 390         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 391
 392         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 393         final String rd = vrfTableKey.getRouteDistinguisher();
 394         if (vpnToDpnList != null) {
 395             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 396                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 397                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 398                 }
 399             }
 400         }
 401     }
 402
 403     // FIXME: Refactoring needed here.
 404     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 405     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 406                                    int addOrRemove) {
 407         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 408         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 409
 410         String rd = vrfTableKey.getRouteDistinguisher();
 411         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 412         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 413             if (vpnInstance == null) {
 414                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 415                 return;
 416             }
 417         } else {
 418             Preconditions.checkNotNull(vpnInstance,
 419                                        "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 420         }
 421         String vpnUuid = vpnInstance.getVpnInstanceName();
 422         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 423                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 424
 425         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 426         // there an interVpnLink for the involved vpn in order to make learn the new route to
 427         // the other part of the inter-vpn-link.
 428
 429         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 430         Optional<InterVpnLink> interVpnLink =
 431             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 432                                                   : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 433         if ( !interVpnLink.isPresent() ) {
 434             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 435             return;
 436         }
 437
 438         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 439         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 440         boolean proceed =
 441             (addOrRemove == NwConstants.DEL_FLOW) || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 442                                                        && interVpnLink.get().isBgpRoutesLeaking() );
 443
 444         if ( proceed ) {
 445             String theOtherVpnId = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid)
 446                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 447                     : vpnUuid;
 448
 449             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 450             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 451
 452             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 453                     InstanceIdentifier.builder(FibEntries.class)
 454                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 455                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 456                             .build();
 457             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 458                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 459                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 460                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 461                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 462                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 463                         .setLabel(label)
 464                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 465                         .build();
 466                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 467             } else {
 468                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 469                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 470             }
 471         }
 472     }
 473
 474     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 475         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 476         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 477         prefixBuilder.setDpnId(lri.getDpnId());
 478         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 479         prefixBuilder.setIpAddress(lri.getPrefix());
 480         // Increment the refCount here
 481         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 482                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 483         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 484         if (!isPresentInList) {
 485             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 486             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 487             vpnInstanceNames.add(vpnInstanceName);
 488             builder.setVpnInstanceList(vpnInstanceNames);
 489             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 490         } else {
 491             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 492         }
 493         return prefixBuilder.build();
 494     }
 495
 496     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 497                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 498         Boolean wrTxPresent = true;
 499         if (tx == null) {
 500             wrTxPresent = false;
 501             tx = dataBroker.newWriteOnlyTransaction();
 502         }
 503         synchronized (vrfEntry.getLabel().toString().intern()) {
 504             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 505             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 506                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 507
 508                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 509                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 510                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 511                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 512                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 513                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 514                         }
 515                     }
 516                 }
 517                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 518                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 519             }
 520         }
 521         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 522         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 523         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 524         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 525         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 526
 527         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 528             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 529             // reinitialize instructions list for LFIB Table
 530             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 531
 532             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 533             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 534             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 535             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 536
 537             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 538         }
 539         if (!wrTxPresent ) {
 540             tx.submit();
 541         }
 542     }
 543
 544     public void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 545         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 546         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 547         // packet is commuted from Vpn2 to Vpn1.
 548         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 549         if ( !vpnNameOpc.isPresent() ) {
 550             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 551             return;
 552         }
 553
 554         String vpnName = vpnNameOpc.get();
 555         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 556         boolean interVpnLinkFound = false;
 557         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 558             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 559             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 560                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 561             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 562                 interVpnLinkFound = true;
 563
 564                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 565                 if ( !vpnLinkState.isPresent()
 566                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 567                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 568                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 569                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 570                     return;
 571                 }
 572
 573                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 574                                                                : vpnLinkState.get().getSecondEndpointState().getDpId();
 575                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 576                                                  : vpnLinkState.get().getFirstEndpointState().getLportTag();
 577
 578                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 579
 580                 for ( BigInteger dpId : targetDpns ) {
 581                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 582
 583                     BigInteger[] metadata = new BigInteger[] {
 584                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 585                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 586                     };
 587                     List<InstructionInfo> instructions =
 588                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 589                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 590                                     new InstructionInfo(InstructionType.goto_table,
 591                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 592
 593                     LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
 594                               vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 595                               dpId, interVpnLink.getName());
 596
 597                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 598                             NwConstants.ADD_FLOW, null);
 599                 }
 600
 601                 break;
 602             }
 603         }
 604
 605         if ( !interVpnLinkFound ) {
 606             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 607                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 608         }
 609     }
 610
 611
 612
 613     private void installRouteInInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 614                                             final VrfEntry vrfEntry, long vpnTag) {
 615         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 616         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 617                 && vrfEntry.getNextHopAddressList().size() == 1);
 618         String destination = vrfEntry.getDestPrefix();
 619         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 620         String iVpnLinkName = interVpnLink.getInterVpnLinkName();
 621
 622         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 623         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 624         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 625         if ( interVpnLink.getState().or(State.Error) != State.Active ) {
 626             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 627                     destination, nextHop, iVpnLinkName);
 628             return;
 629         }
 630
 631         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 632         if ( !optOtherEndpointLportTag.isPresent() ) {
 633             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 634                      vpnUuid, iVpnLinkName);
 635             return;
 636         }
 637
 638         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 639         if ( targetDpns.isEmpty() ) {
 640             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, iVpnLinkName);
 641             return;
 642         }
 643
 644         BigInteger[] metadata = new BigInteger[] {
 645                 MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 646                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 647                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 648         };
 649         List<Instruction> instructions =
 650                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 651                               new InstructionInfo(InstructionType.goto_table,
 652                                                   new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 653
 654         String values[] = destination.split("/");
 655         String destPrefixIpAddress = values[0];
 656         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 657
 658         List<MatchInfo> matches = new ArrayList<>();
 659         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 660                                                                               MetaDataUtil.METADATA_MASK_VRFID }));
 661         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 662
 663         if (prefixLength != 0) {
 664             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 665                                       new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 666         }
 667
 668         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 669         String flowRef = getInterVpnFibFlowRef(iVpnLinkName, destination, nextHop);
 670         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 671                                                  COOKIE_VM_FIB_TABLE, matches, instructions);
 672
 673         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}" ,
 674                   vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 675
 676         for ( BigInteger dpId : targetDpns ) {
 677
 678             LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB",
 679                      vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
 680                      dpId, interVpnLink.getInterVpnLinkName());
 681
 682             mdsalManager.installFlow(dpId, flowEntity);
 683         }
 684     }
 685
 686
 687     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 688                                                      InstanceIdentifier<T> path) {
 689
 690         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 691
 692         Optional<T> result = Optional.absent();
 693         try {
 694             result = tx.read(datastoreType, path).get();
 695         } catch (Exception e) {
 696             throw new RuntimeException(e);
 697         }
 698
 699         return result;
 700     }
 701
 702     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 703         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 704         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 705         String localNextHopIP = vrfEntry.getDestPrefix();
 706
 707         if (localNextHopInfo == null) {
 708             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 709             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 710             if (extraRoute != null) {
 711                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 712                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 713                     if (nextHopIp != null) {
 714                         localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
 715                         localNextHopIP = nextHopIp + "/32";
 716                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 717                         returnLocalDpnId.add(dpnId);
 718                     }
 719                 }
 720             }
 721             if (localNextHopInfo == null) {
 722             /* imported routes case */
 723                 synchronized (vrfEntry.getLabel().toString().intern()) {
 724                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 725                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 726                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 727                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 728                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 729                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 730                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 731                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 732                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 733                                     localNextHopIP = lri.getPrefix();
 734                                 } else {
 735                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 736                                     localNextHopIP = lri.getPrefix();
 737                                 }
 738                             }
 739                             if (localNextHopInfo != null) {
 740                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 741                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 742                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 743                                 returnLocalDpnId.add(dpnId);
 744                             }
 745                         }
 746                     }
 747                 }
 748             }
 749         } else {
 750             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 751             returnLocalDpnId.add(dpnId);
 752         }
 753
 754         return returnLocalDpnId;
 755     }
 756
 757     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 758                                                 final VrfEntry vrfEntry, Long parentVpnId){
 759         if (localNextHopInfo != null) {
 760             final BigInteger dpnId = localNextHopInfo.getDpnId();
 761             if (!isVpnPresentInDpn(rd, dpnId)) {
 762                 LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
 763                 return BigInteger.ZERO;
 764             }
 765
 766             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 767                     localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix());
 768             if (groupId == 0) {
 769                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 770                         vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 771                 return BigInteger.ZERO;
 772             }
 773             List<ActionInfo> actionsInfos =
 774                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 775             final List<InstructionInfo> instructions =
 776                     Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 777             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 778                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 779             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 780             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 781                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 782                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 783             } else {
 784                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 785             }
 786             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 787             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 788                     new Callable<List<ListenableFuture<Void>>>() {
 789                         @Override
 790                         public List<ListenableFuture<Void>> call() throws Exception {
 791                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 792                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 793                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 794                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 795                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 796                             }
 797                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 798                             futures.add(tx.submit());
 799                             return futures;
 800                         }
 801                     });
 802             return dpnId;
 803         }
 804         return BigInteger.ZERO;
 805     }
 806
 807     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 808         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 809         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 810         if (dpnInVpn.isPresent()) {
 811             return true;
 812         }
 813         return false;
 814     }
 815
 816     private LabelRouteInfo getLabelRouteInfo(Long label) {
 817         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 818                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 819         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 820         if (opResult.isPresent()) {
 821             return opResult.get();
 822         }
 823         return null;
 824     }
 825
 826     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 827         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 828         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 829                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 830         if (lri == null) {
 831             return true;
 832         }
 833         List<String> vpnInstancesList = lri.getVpnInstanceList() != null ? lri.getVpnInstanceList() : new ArrayList<String>();
 834         if (vpnInstancesList.contains(vpnInstanceName)) {
 835             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 836             vpnInstancesList.remove(vpnInstanceName);
 837         }
 838         if (vpnInstancesList.size() == 0) {
 839             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 840             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 841             return true;
 842         } else {
 843             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 844             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 845             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 846         }
 847         return false;
 848     }
 849
 850     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 851                                       WriteTransaction tx) {
 852         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 853         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 854
 855
 856         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 857
 858         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 859                 dpId, label, groupId);
 860     }
 861
 862     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 863                                                  WriteTransaction tx) {
 864         List<MatchInfo> mkMatches = new ArrayList<>();
 865
 866         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 867
 868         // Matching metadata
 869         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 870         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 871
 872         List<InstructionInfo> mkInstructions = new ArrayList<>();
 873         mkInstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 874
 875         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 876                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 877                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 878
 879         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 880
 881         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 882
 883         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 884         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 885                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 886                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 887         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 888     }
 889
 890     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 891         FlowEntity flowEntity;
 892         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 893         List<MatchInfo> mkMatches = new ArrayList<>();
 894         // Matching metadata
 895         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 896         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 897                 NwConstants.INTERNAL_TUNNEL_TABLE,
 898                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 899                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 900                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 901         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 902         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 903         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 904                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 905                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 906
 907         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 908         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 909     }
 910
 911     /**
 912      * Delete local FIB entry
 913      * @param vpnId
 914      * @param rd
 915      * @param vrfEntry
 916      * @return
 917      */
 918     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 919         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 920         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 921         String localNextHopIP = vrfEntry.getDestPrefix();
 922
 923         if (localNextHopInfo == null) {
 924             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 925             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 926             if (extra_route != null) {
 927                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 928                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 929                     if (nextHopIp != null) {
 930                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 931                         localNextHopIP = nextHopIp + "/32";
 932                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 933                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 934                         if (!dpnId.equals(BigInteger.ZERO)) {
 935                             returnLocalDpnId.add(dpnId);
 936                         }
 937                     }
 938                 }
 939             }
 940
 941             if (localNextHopInfo == null) {
 942               /* Imported VRF entry */
 943                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 944                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 945                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 946                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
 947                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
 948                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
 949                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 950                     if (!dpnId.equals(BigInteger.ZERO)) {
 951                         returnLocalDpnId.add(dpnId);
 952                     }
 953                 }
 954             }
 955
 956
 957         } else {
 958             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 959                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 960             if (!dpnId.equals(BigInteger.ZERO)) {
 961                 returnLocalDpnId.add(dpnId);
 962             }
 963         }
 964
 965         return returnLocalDpnId;
 966     }
 967
 968     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
 969                                                 final Long vpnId, final String rd,
 970                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
 971         if (localNextHopInfo != null) {
 972             final BigInteger dpnId = localNextHopInfo.getDpnId();;
 973             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 974             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 975                     new Callable<List<ListenableFuture<Void>>>() {
 976                         @Override
 977                         public List<ListenableFuture<Void>> call() throws Exception {
 978                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 979                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
 980                                     NwConstants.DEL_FLOW, tx);
 981                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 982                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
 983                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
 984                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
 985                             }
 986                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 987                             futures.add(tx.submit());
 988                             return futures;
 989                         }
 990                     });
 991             //TODO: verify below adjacency call need to be optimized (?)
 992             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
 993             return dpnId;
 994         }
 995         return BigInteger.ZERO;
 996     }
 997
 998     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
 999         return InstanceIdentifier.builder(VpnToExtraroute.class)
1000                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1001                         new ExtrarouteKey(ipPrefix)).build();
1002     }
1003
1004     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1005         Optional<Extraroute> extraRouteInfo =
1006                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1007         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1008
1009     }
1010
1011     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1012         try {
1013             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1014                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1015             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1016             if(!rpcResult.isSuccessful()) {
1017                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1018             } else {
1019                 return rpcResult.getResult().getTunnelType();
1020             }
1021
1022         } catch (InterruptedException | ExecutionException e) {
1023             LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
1024         }
1025
1026         return null;
1027
1028     }
1029     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1030                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1031         Boolean wrTxPresent = true;
1032         if (tx == null) {
1033             wrTxPresent = false;
1034             tx = dataBroker.newWriteOnlyTransaction();
1035         }
1036         String rd = vrfTableKey.getRouteDistinguisher();
1037         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1038                 vrfEntry.getDestPrefix(), rd, tx);
1039         /********************************************/
1040         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1041
1042         if (adjacencyResults.isEmpty()) {
1043             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1044                     vrfEntry.getNextHopAddressList(), rd);
1045             LOG.warn("Failed to add Route: {} in vpn: {}",
1046                     vrfEntry.getDestPrefix(), rd);
1047             return;
1048         }
1049
1050         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1051             List<InstructionInfo> instructions = new ArrayList<>();
1052             List<ActionInfo> actionInfos = new ArrayList<>();
1053             String egressInterface = adjacencyResult.getInterfaceName();
1054             if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
1055                 addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
1056             } else {
1057                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1058             }
1059             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface);
1060             if (egressActions.isEmpty()) {
1061                 LOG.error(
1062                         "Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.",
1063                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface);
1064                 return;
1065             }
1066             actionInfos.addAll(egressActions);
1067             instructions.add(new InstructionInfo(InstructionType.apply_actions, actionInfos));
1068             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1069         }
1070         if(!wrTxPresent ){
1071             tx.submit();
1072         }
1073         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1074     }
1075
1076     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1077         String ipPrefix = vrfEntry.getDestPrefix();
1078         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1079         if (prefixInfo == null) {
1080             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1081             return;
1082         }
1083
1084         String ifName = prefixInfo.getVpnInterfaceName();
1085         if (ifName == null) {
1086             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1087             return;
1088         }
1089
1090         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1091         if (macAddress == null) {
1092             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1093             return;
1094         }
1095
1096         actionInfos.add(new ActionInfo(ActionType.set_field_eth_dest, new String[] { macAddress }, actionInfos.size()));
1097     }
1098
1099     private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
1100             List<ActionInfo> actionInfos) {
1101         Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1102         if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1103             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1104             actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[] { null }));
1105             actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label,
1106                     new String[] { Long.toString(vrfEntry.getLabel()) }));
1107         } else {
1108             int label = vrfEntry.getLabel().intValue();
1109             BigInteger tunnelId;
1110             // FIXME vxlan vni bit set is not working properly with OVS.need to
1111             // revisit
1112             if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1113                 tunnelId = BigInteger.valueOf(label);
1114             } else {
1115                 tunnelId = BigInteger.valueOf(label);
1116             }
1117
1118             LOG.debug("adding set tunnel id action for label {}", label);
1119             actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[] { tunnelId }));
1120             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1121         }
1122     }
1123
1124     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1125         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1126         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1127         if (dpnInVpn.isPresent()) {
1128             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1129             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1130
1131             if (vpnInterfaces.remove(currVpnInterface)) {
1132                 if (vpnInterfaces.isEmpty()) {
1133                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1134                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1135                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1136                 } else {
1137                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1138                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1139                             VpnInterfaces.class,
1140                             new VpnInterfacesKey(intfName)));
1141                 }
1142             }
1143         }
1144     }
1145
1146     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1147     /* Get interface info from prefix to interface mapping;
1148         Use the interface info to get the corresponding vpn interface op DS entry,
1149         remove the adjacency corresponding to this fib entry.
1150         If adjacency removed is the last adjacency, clean up the following:
1151          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1152          - prefix to interface entry
1153          - vpn interface op DS
1154      */
1155         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1156         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1157         Extraroute extraRoute = null;
1158         if (prefixInfo == null) {
1159             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1160             if(extraRoute != null) {
1161                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1162                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1163
1164                     if (nextHopIp != null) {
1165                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
1166                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1167                     }
1168                 }
1169             }
1170             if (prefixInfo == null) {
1171                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1172                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1173                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1174                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1175                     prefixBuilder.setDpnId(lri.getDpnId());
1176                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1177                     prefixBuilder.setIpAddress(lri.getPrefix());
1178                     prefixInfo = prefixBuilder.build();
1179                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1180                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1181                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1182                 }
1183             }
1184         } else {
1185             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1186         }
1187     }
1188
1189     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1190                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1191
1192         if (prefixInfo == null) {
1193             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1194             return; //Don't have any info for this prefix (shouldn't happen); need to return
1195         }
1196
1197         String ifName = prefixInfo.getVpnInterfaceName();
1198         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1199         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1200                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1201     }
1202
1203     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1204         Prefixes prefixInfo;
1205         Long vpnId;
1206         String rd;
1207         VrfEntry vrfEntry;
1208         Extraroute extraRoute;
1209
1210         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1211                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1212             this.prefixInfo = prefixInfo;
1213             this.vpnId = vpnId;
1214             this.rd= rd;
1215             this.vrfEntry= vrfEntry;
1216             this.extraRoute = extraRoute;
1217         }
1218
1219         @Override
1220         public List<ListenableFuture<Void>> call() throws Exception {
1221             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1222             // to call the respective helpers.
1223
1224             //First Cleanup LabelRouteInfo
1225             synchronized (vrfEntry.getLabel().toString().intern()) {
1226                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1227                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1228                                 vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1229                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1230                     String vpnInstanceName = "";
1231                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1232                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1233                         }
1234                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1235                     if (lriRemoved) {
1236                             String parentRd = lri.getParentVpnRd();
1237                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1238                                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1239                         }
1240                 } else {
1241                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1242                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1243                 }
1244             }
1245             String ifName = prefixInfo.getVpnInterfaceName();
1246             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1247                     FibUtil.getVpnInterfaceIdentifier(ifName));
1248             if (optvpnInterface.isPresent()) {
1249                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1250                 if (vpnId != associatedVpnId) {
1251                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1252                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1253                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1254                     return null;
1255                 } else {
1256                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1257                             vrfEntry.getDestPrefix(), associatedVpnId);
1258                 }
1259             }
1260             if (extraRoute != null) {
1261                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1262                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1263             }
1264             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1265                     FibUtil.getAdjListPath(ifName));
1266             int numAdj = 0;
1267             if (optAdjacencies.isPresent()) {
1268                 numAdj = optAdjacencies.get().getAdjacency().size();
1269             }
1270             //remove adjacency corr to prefix
1271             if (numAdj > 1) {
1272                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1273                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1274                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1275             }
1276             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1277                 //clean up the vpn interface from DpnToVpn list
1278                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1279                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1280                         FibUtil.getVpnInterfaceIdentifier(ifName));
1281             }
1282             return null;
1283         }
1284     }
1285
1286     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1287         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1288         final String rd  = vrfTableKey.getRouteDistinguisher();
1289         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1290         if (vpnInstance == null) {
1291             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1292             return;
1293         }
1294         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1295         long elanTag = 0L;
1296         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1297         if (subnetRoute != null) {
1298             elanTag = subnetRoute.getElantag();
1299             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1300                     rd, vrfEntry.getDestPrefix(), elanTag);
1301             if (vpnToDpnList != null) {
1302                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1303                 dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1304                         new Callable<List<ListenableFuture<Void>>>() {
1305                             @Override
1306                             public List<ListenableFuture<Void>> call() throws Exception {
1307                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1308
1309                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1310
1311                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1312                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1313                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1314                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1315                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1316                                     }
1317                                 }
1318                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1319                                 futures.add(tx.submit());
1320                                 return futures;
1321                             }
1322                         });
1323             }
1324             synchronized (vrfEntry.getLabel().toString().intern()) {
1325                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1326                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1327                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1328                     String vpnInstanceName = "";
1329                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1330                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1331                     }
1332                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1333                     if (lriRemoved) {
1334                         String parentRd = lri.getParentVpnRd();
1335                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1336                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1337                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1338                                 vrfEntry.getDestPrefix());
1339                     }
1340                 } else {
1341                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1342                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1343                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1344                             vrfEntry.getDestPrefix());
1345                 }
1346             }
1347             return;
1348         }
1349         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1350             return;
1351         }
1352
1353         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1354                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1355         if (vpnToDpnList != null) {
1356             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1357             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1358                     new Callable<List<ListenableFuture<Void>>>() {
1359                         @Override
1360                         public List<ListenableFuture<Void>> call() throws Exception {
1361                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1362
1363                             if (localDpnIdList.size() <= 0) {
1364                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1365                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1366                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1367                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1368                                         }
1369                                     } else {
1370                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1371                                     }
1372                                 }
1373                             } else {
1374                                 for (BigInteger localDpnId : localDpnIdList) {
1375                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1376                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1377                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1378                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1379                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1380                                                 }
1381                                             } else {
1382                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1383                                             }
1384                                         }
1385                                     }
1386                                 }
1387                             }
1388                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1389                             futures.add(tx.submit());
1390                             return futures;
1391                         }
1392                     });
1393         }
1394
1395         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1396         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1397         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1398
1399         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1400         // of the interVpnLink.
1401         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1402         if ( optVpnUuid.isPresent() ) {
1403             String vpnUuid = optVpnUuid.get();
1404             List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
1405             if(routeNexthoplist.isEmpty()) {
1406                 LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
1407                 return;
1408             }
1409             String routeNexthop = routeNexthoplist.get(0);
1410             Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1411             if ( optInterVpnLink.isPresent() ) {
1412                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1413                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid))
1414                 {
1415                     // This is route that points to the other endpoint of an InterVpnLink
1416                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1417                     removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1418                                                  interVpnLink.isFirstEndpointVpnName(rd),
1419                                                  vrfEntry);
1420                 }
1421             }
1422         }
1423
1424     }
1425
1426     /*
1427       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1428       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1429       provided by ResourceBatchingManager
1430      */
1431     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1432         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1433
1434         final String rd  = vrfTableKey.getRouteDistinguisher();
1435         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1436         if (vpnInstance == null) {
1437             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1438             return;
1439         }
1440         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1441         if (vpnToDpnList != null) {
1442             for (VpnToDpnList curDpn : vpnToDpnList) {
1443                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1444                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1445                 }
1446             }
1447         }
1448     }
1449
1450     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1451                                   final long vpnId, final VrfTablesKey vrfTableKey,
1452                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1453
1454         Boolean wrTxPresent = true;
1455         if (tx == null) {
1456             wrTxPresent = false;
1457             tx = dataBroker.newWriteOnlyTransaction();
1458         }
1459
1460         LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1461         String rd = vrfTableKey.getRouteDistinguisher();
1462
1463         if(localDpnId != null) {
1464             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1465             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1466             return;
1467         }
1468
1469         // below two reads are kept as is, until best way is found to identify dpnID
1470         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1471         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1472
1473         if (localNextHopInfo == null && extraRoute != null) {
1474             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1475             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1476                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1477                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1478             }
1479         } else {
1480             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1481         }
1482         if(!wrTxPresent ){
1483             tx.submit();
1484         }
1485     }
1486
1487     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1488                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1489         boolean isRemoteRoute = true;
1490         if (localNextHopInfo != null) {
1491             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1492         }
1493         if (isRemoteRoute) {
1494             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1495             return true;
1496         } else {
1497             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1498             return false;
1499         }
1500     }
1501
1502     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1503         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1504         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1505     }
1506
1507     private long get
1508             (byte[] rawIpAddress) {
1509         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1510                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1511     }
1512
1513     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1514                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1515         Boolean wrTxPresent = true;
1516         if (tx == null) {
1517             wrTxPresent = false;
1518             tx = dataBroker.newWriteOnlyTransaction();
1519         }
1520
1521         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1522         String values[] = vrfEntry.getDestPrefix().split("/");
1523         String ipAddress = values[0];
1524         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1525         if (addOrRemove == NwConstants.ADD_FLOW) {
1526             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1527         } else {
1528             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1529         }
1530         InetAddress destPrefix;
1531         try {
1532             destPrefix = InetAddress.getByName(ipAddress);
1533         } catch (UnknownHostException e) {
1534             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1535             return;
1536         }
1537
1538         List<MatchInfo> matches = new ArrayList<>();
1539
1540         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1541                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1542
1543         matches.add(new MatchInfo(MatchFieldType.eth_type,
1544                 new long[] { NwConstants.ETHTYPE_IPV4 }));
1545
1546         if(prefixLength != 0) {
1547             matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1548                     destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1549         }
1550         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1551         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1552         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1553                 COOKIE_VM_FIB_TABLE, matches, instructions);
1554
1555         Flow flow = flowEntity.getFlowBuilder().build();
1556         String flowId = flowEntity.getFlowId();
1557         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1558         Node nodeDpn = buildDpnNode(dpId);
1559
1560         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1561                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1562                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1563
1564         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1565             SubTransaction subTransaction = new SubTransactionImpl();
1566             if (addOrRemove == NwConstants.ADD_FLOW) {
1567                 subTransaction.setInstanceIdentifier(flowInstanceId);
1568                 subTransaction.setInstance(flow);
1569                 subTransaction.setAction(SubTransaction.CREATE);
1570             } else {
1571                 subTransaction.setInstanceIdentifier(flowInstanceId);
1572                 subTransaction.setAction(SubTransaction.DELETE);
1573             }
1574             transactionObjects.add(subTransaction);
1575         }
1576
1577         if (addOrRemove == NwConstants.ADD_FLOW) {
1578             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1579         } else {
1580             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1581         }
1582
1583         if(!wrTxPresent ){
1584             tx.submit();
1585         }
1586     }
1587
1588     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1589     private Node buildDpnNode(BigInteger dpnId) {
1590         NodeId nodeId = new NodeId("openflow:" + dpnId);
1591         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1592
1593         return nodeDpn;
1594     }
1595
1596     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1597                                     int addOrRemove, WriteTransaction tx) {
1598         Boolean wrTxPresent = true;
1599         if (tx == null) {
1600             wrTxPresent = false;
1601             tx = dataBroker.newWriteOnlyTransaction();
1602         }
1603
1604         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1605         matches.add(new MatchInfo(MatchFieldType.eth_type,
1606                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1607         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1608
1609         // Install the flow entry in L3_LFIB_TABLE
1610         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1611
1612         FlowEntity flowEntity;
1613         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1614                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1615         Flow flow = flowEntity.getFlowBuilder().build();
1616         String flowId = flowEntity.getFlowId();
1617         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1618         Node nodeDpn = buildDpnNode(dpId);
1619         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1620                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1621                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1622
1623         if (addOrRemove == NwConstants.ADD_FLOW) {
1624             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1625         } else {
1626             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1627         }
1628         if(!wrTxPresent ){
1629             tx.submit();
1630         }
1631
1632         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1633                   dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1634     }
1635
1636     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1637                                       final String ipPrefixAddress) {
1638         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1639         try {
1640             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1641         } catch (NullPointerException e) {
1642             LOG.trace("", e);
1643         }
1644     }
1645
1646     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1647             final FutureCallback<List<Void>> callback) {
1648         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1649         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1650         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1651         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1652         if (!vrfTable.isPresent()) {
1653             LOG.warn("VRF Table not yet available for RD {}", rd);
1654             if (callback != null) {
1655                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1656                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1657                 Futures.addCallback(listenableFuture, callback);
1658             }
1659             return;
1660         }
1661         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1662         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1663                 new Callable<List<ListenableFuture<Void>>>() {
1664             @Override
1665             public List<ListenableFuture<Void>> call() throws Exception {
1666                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1667                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1668                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1669                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1670                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1671                         if (subnetRoute != null) {
1672                             long elanTag = subnetRoute.getElantag();
1673                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1674                             continue;
1675                         }
1676                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1677                         if (routerInt != null) {
1678                             LOG.trace( "Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1679                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1680                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(),
1681                                     new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1682                             continue;
1683                         }
1684                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
1685                             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1686                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1687                                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1688                                 if (lri.getDpnId().equals(dpnId)) {
1689                                     createLocalFibEntry(vpnId, rd, vrfEntry);
1690                                     continue;
1691                                 }
1692                             }
1693                         }
1694                         // Passing null as we don't know the dpn
1695                         // to which prefix is attached at this point
1696                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1697                     }
1698                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1699                     futures.add(tx.submit());
1700                 }
1701                 if (callback != null) {
1702                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1703                     Futures.addCallback(listenableFuture, callback);
1704                 }
1705                 return futures;
1706             }
1707         });
1708     }
1709
1710
1711     public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1712                                  final String localNextHopIp, final String remoteNextHopIp) {
1713         LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1714                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1715         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1716         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1717         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1718         if (vrfTable.isPresent()) {
1719             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1720             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1721                     new Callable<List<ListenableFuture<Void>>>() {
1722                         @Override
1723                         public List<ListenableFuture<Void>> call() throws Exception {
1724                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1725                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1726                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1727                                 LOG.trace("populate FIB starts on Dpn " + dpnId
1728                                         + "rd  " + rd.toString()
1729                                         + "localNextHopIp " + localNextHopIp
1730                                         + "remoteNextHopIp" + remoteNextHopIp
1731                                         + "vpnId " + vpnId );
1732                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1733                                     LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1734
1735                                     if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1736                                         if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1737                                             LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1738                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1739                                         }
1740                                     } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
1741                                         BigInteger dpnIdForPrefix = null;
1742                                         String destPfx = vrfEntry.getDestPrefix();
1743                                         if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
1744                                             Optional<Extraroute> extraRouteInfo =
1745                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1746                                                             getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1747                                             if (extraRouteInfo.isPresent()) {
1748                                                 continue;
1749                                             }
1750                                             dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1751                                         } else {
1752                                             // Subnet Route handling
1753                                             Optional<Prefixes> localNextHopInfoData =
1754                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1755                                                             FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
1756                                             if (localNextHopInfoData.isPresent()) {
1757                                                 Prefixes prefixes = localNextHopInfoData.get();
1758                                                 dpnIdForPrefix = prefixes.getDpnId();
1759                                             }
1760                                         }
1761                                         if (dpnIdForPrefix == null) {
1762                                             LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1763                                                     vrfEntry.getDestPrefix());
1764                                             continue;
1765                                         }
1766                                         int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1767                                         if (sameDpnId != 0) {
1768                                             LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1769                                                     dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1770                                             continue;
1771                                         }
1772                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1773                                         List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
1774                                         newNextHopAddrList.add(localNextHopIp);
1775                                         VrfEntry newVrfEntry =
1776                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
1777                                         // Just update the VrfEntry
1778                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1779                                                 vrfEntryId, newVrfEntry);
1780                                         // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
1781                                         //       vrfEntryId, newVrfEntry);
1782                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1783                                         LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1784                                     }
1785                                 }
1786                                 futures.add(writeTransaction.submit());
1787                                 LOG.trace("populate FIB ends on Dpn " + dpnId
1788                                         + "rd  " + rd.toString()
1789                                         + "localNextHopIp " + localNextHopIp
1790                                         + "remoteNextHopIp" + remoteNextHopIp
1791                                         + "vpnId " + vpnId);
1792                             }
1793                             return futures;
1794                         }
1795                     });
1796         }
1797     }
1798
1799     public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
1800                                   final long vpnId, final String  rd, final String destPrefix ,
1801                                   final String localNextHopIP, final String remoteNextHopIp) {
1802
1803         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1804         dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1805                         + "local dpid" + localDpnId
1806                         + "remote dpid" + remoteDpnId
1807                         + "vpnId" + vpnId
1808                         + "localNHIp" + localNextHopIP
1809                         + "remoteNHIp" + remoteNextHopIp,
1810                 new Callable<List<ListenableFuture<Void>>>() {
1811                     @Override
1812                     public List<ListenableFuture<Void>> call() throws Exception {
1813                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1814                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1815                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1816                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1817                         if (vrfEntry == null) {
1818                             return futures;
1819                         }
1820                         LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1821                                         "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1822                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1823                         if (action == true) {
1824                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1825                             LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1826                             createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1827                         } else {
1828                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1829                             LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1830                             deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1831                         }
1832                         futures.add(writeTransaction.submit());
1833                         return futures;
1834                     }
1835                 });
1836     }
1837
1838     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1839                                  final FutureCallback<List<Void>> callback) {
1840         LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1841         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1842         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1843         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1844         if (vrfTable.isPresent()) {
1845             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1846             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1847                     new Callable<List<ListenableFuture<Void>>>() {
1848                         @Override
1849                         public List<ListenableFuture<Void>> call() throws Exception {
1850                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1851                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1852                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1853                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1854                                     /* Handle subnet routes here */
1855                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1856                                     if (subnetRoute != null) {
1857                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1858                                                 dpnId, rd);
1859                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1860                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1861                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1862                                                 vrfEntry.getDestPrefix());
1863                                         continue;
1864                                     }
1865                                     // ping responder for router interfaces
1866                                     RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1867                                     if (routerInt != null) {
1868                                         LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
1869                                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1870                                         installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(),
1871                                                 new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
1872                                         continue;
1873                                     }
1874                                     // Passing null as we don't know the dpn
1875                                     // to which prefix is attached at this point
1876                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1877                                 }
1878                                 futures.add(tx.submit());
1879                                 if (callback != null) {
1880                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1881                                     Futures.addCallback(listenableFuture, callback);
1882                                 }
1883                             }
1884                             return futures;
1885                         }
1886
1887                     });
1888         }
1889     }
1890
1891     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1892                                  final String localNextHopIp, final String remoteNextHopIp,
1893                                  final FutureCallback<List<Void>> callback) {
1894         LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1895                         " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1896                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1897         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1898         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1899         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1900         if (vrfTable.isPresent()) {
1901             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1902             dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
1903                     new Callable<List<ListenableFuture<Void>>>() {
1904                         @Override
1905                         public List<ListenableFuture<Void>> call() throws Exception {
1906                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1907                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1908                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1909                                 LOG.trace("cleanup FIB starts on Dpn " + dpnId
1910                                         + "rd  " + rd.toString()
1911                                         + "localNextHopIp " + localNextHopIp
1912                                         + "remoteNextHopIp" + remoteNextHopIp
1913                                         + "vpnId " + vpnId);
1914
1915                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1916                                     LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1917                                     if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1918                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1919                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1920                                     }
1921
1922                                     if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1923                                         LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1924                                                 vrfEntry.getDestPrefix(), dpnId);
1925                                         String destPfx = vrfEntry.getDestPrefix();
1926                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1927                                         List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1928                                         newList.remove(localNextHopIp);
1929                                         VrfEntry newVrfEntry =
1930                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1931                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1932                                                 vrfEntryId, newVrfEntry);
1933                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1934                                         LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1935                                     }
1936                                 }
1937                                 futures.add(writeTransaction.submit());
1938                                 if (callback != null) {
1939                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1940                                     Futures.addCallback(listenableFuture, callback);
1941                                 }
1942                                 LOG.trace("cleanup FIB ends on Dpn " + dpnId
1943                                         + "rd  " + rd.toString()
1944                                         + "localNextHopIp " + localNextHopIp
1945                                         + "remoteNextHopIp" + remoteNextHopIp
1946                                         + "vpnId " + vpnId);
1947                             }
1948                             return futures;
1949                         }
1950                     });
1951
1952         }
1953     }
1954
1955     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1956         InstanceIdentifierBuilder<VrfTables> idBuilder =
1957                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1958         InstanceIdentifier<VrfTables> id = idBuilder.build();
1959         return id;
1960     }
1961
1962     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1963         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1964                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1965                 .append(priority).toString();
1966     }
1967
1968     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1969         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1970                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1971                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1972                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1973                 .append(destPrefix.getHostAddress()).toString();
1974     }
1975
1976     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1977         return new StringBuilder(64).append(FLOWID_PREFIX)
1978                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1979                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1980                 .append(nextHop).toString();
1981     }
1982
1983     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
1984                                             final VrfEntry vrfEntry, String rd) {
1985         List<AdjacencyResult> adjacencyList = new ArrayList<>();
1986         List<String> prefixIpList = new ArrayList<>();
1987         LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
1988                 remoteDpnId, vpnId, vrfEntry);
1989         try {
1990             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1991                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1992                 if (extra_route == null) {
1993                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1994                 } else {
1995                     prefixIpList = new ArrayList<>();
1996                     for (String extraRouteIp : extra_route.getNexthopIpList()) {
1997                         prefixIpList.add(extraRouteIp + "/32");
1998                     }
1999                 }
2000             } else {
2001                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
2002             }
2003
2004             for (String prefixIp : prefixIpList) {
2005                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
2006                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
2007                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2008                             prefixIp, nextHopIp);
2009                     if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2010                         adjacencyList.add(adjacencyResult);
2011                     }
2012                 }
2013             }
2014         } catch (NullPointerException e) {
2015             LOG.trace("", e);
2016         }
2017         return adjacencyList;
2018     }
2019
2020     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2021         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2022                 InstanceIdentifier.create(VpnInstanceOpData.class)
2023                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2024         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2025                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2026         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2027     }
2028
2029     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2030         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2031                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2032                 .append(FLOWID_PREFIX).toString();
2033     }
2034
2035     /*
2036      * Install flow entry in protocol table to forward mpls
2037      * coming through gre tunnel to LFIB table.
2038      */
2039     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2040         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2041         // Instruction to goto L3 InterfaceTable
2042         List<InstructionInfo> instructions = new ArrayList<>();
2043         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2044         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2045         matches.add(new MatchInfo(MatchFieldType.eth_type,
2046                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2047         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2048                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2049                         NwConstants.L3_LFIB_TABLE),
2050                 DEFAULT_FIB_FLOW_PRIORITY,
2051                 "Protocol Table For LFIB",
2052                 0, 0,
2053                 COOKIE_PROTOCOL_TABLE,
2054                 matches, instructions);
2055
2056         if (addOrRemove == NwConstants.ADD_FLOW) {
2057             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2058             mdsalManager.installFlow(flowEntityToLfib);
2059         } else {
2060             mdsalManager.removeFlow(flowEntityToLfib);
2061         }
2062     }
2063
2064     public List<String> printFibEntries() {
2065         List<String> result = new ArrayList<>();
2066         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2067         result.add("-------------------------------------------------------------------");
2068         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2069         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2070         if (fibEntries.isPresent()) {
2071             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2072             for (VrfTables vrfTable : vrfTables) {
2073                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2074                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2075                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2076                                 vrfTable.getRouteDistinguisher(),
2077                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2078                     }
2079                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2080                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2081                                 vrfTable.getRouteDistinguisher(),
2082                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2083                     }
2084                 }
2085             }
2086         }
2087         return result;
2088     }
2089
2090
2091     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2092         InstanceIdentifier<VrfEntry> vrfEntryId =
2093                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2094                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2095         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2096         if (vrfEntry.isPresent())  {
2097             return vrfEntry.get();
2098         }
2099         return null;
2100     }
2101
2102     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2103         InstanceIdentifier<VrfEntry> vrfEntryId =
2104                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2105                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2106         return vrfEntryId;
2107     }
2108     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2109             long vpnId, int addOrRemove) {
2110         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2111         if (routerInt != null && vpnToDpnList != null) {
2112             String routerId = routerInt.getUuid();
2113             String macAddress = routerInt.getMacAddress();
2114             String ipValue = routerInt.getIpAddress();
2115             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2116                     routerId, ipValue, macAddress);
2117             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2118                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2119                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2120                             new MacAddress(macAddress), addOrRemove);
2121                 }
2122             }
2123             return true;
2124         }
2125         return false;
2126     }
2127
2128     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2129                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2130         String[] subSplit = routerInternalIp.split("/");
2131
2132         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2133         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2134                 dpnId, routerInternalIp, vpnId, subSplit[0]);
2135
2136         List<MatchInfo> matches = new ArrayList<>();
2137
2138         matches.add(new MatchInfo(MatchFieldType.ip_proto, new long[] { IPProtocols.ICMP.intValue() }));
2139         matches.add(new MatchInfo(MatchFieldType.metadata,
2140                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
2141         matches.add(new MatchInfo(MatchFieldType.icmp_v4, new long[] { (short) 8, (short) 0 }));
2142         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
2143         matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { subSplit[0], "32" }));
2144
2145         List<ActionInfo> actionsInfos = new ArrayList<>();
2146
2147         // Set Eth Src and Eth Dst
2148         actionsInfos.add(new ActionInfo(ActionType.move_src_dst_eth, new String[] {}));
2149         actionsInfos.add(new ActionInfo(ActionType.set_field_eth_src, new String[] { routerMac.getValue() }));
2150
2151         // Move Ip Src to Ip Dst
2152         actionsInfos.add(new ActionInfo(ActionType.move_src_dst_ip, new String[] {}));
2153         actionsInfos.add(new ActionInfo(ActionType.set_source_ip, new String[] { subSplit[0], "32" }));
2154
2155         // Set the ICMP type to 0 (echo reply)
2156         actionsInfos.add(new ActionInfo(ActionType.set_icmp_type, new String[] { "0" }));
2157
2158         actionsInfos.add(new ActionInfo(ActionType.nx_load_in_port, new BigInteger[]{ BigInteger.ZERO }));
2159
2160         actionsInfos.add(new ActionInfo(ActionType.nx_resubmit,
2161                 new String[] { Short.toString(NwConstants.L3_FIB_TABLE) }));
2162
2163         List<InstructionInfo> instructions = new ArrayList<>();
2164
2165         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2166
2167         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2168         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
2169
2170         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2171                 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2172
2173         if (addOrRemove == NwConstants.ADD_FLOW) {
2174             mdsalManager.installFlow(flowEntity);
2175         } else {
2176             mdsalManager.removeFlow(flowEntity);
2177         }
2178     }
2179
2180     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2181                                              final boolean isVpnFirstEndPoint,
2182                                              final VrfEntry vrfEntry)
2183     {
2184         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
2185                                     && vrfEntry.getNextHopAddressList().size() == 1);
2186         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2187
2188         if ( !interVpnLinkState.isPresent()) {
2189             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2190             return;
2191         }
2192
2193         List<BigInteger> targetDpns =
2194             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2195             : interVpnLinkState.get().getSecondEndpointState().getDpId();
2196
2197         String nextHop = vrfEntry.getNextHopAddressList().get(0);
2198
2199         // delete from FIB
2200         //
2201         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2202         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2203         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
2204             .setFlowName(flowRef).build();
2205
2206         LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2207                   interVpnLinkName, flowRef);
2208
2209         for ( BigInteger dpId : targetDpns ) {
2210             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2211                       vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2212                       dpId, interVpnLinkName);
2213
2214             mdsalManager.removeFlow(dpId, flow);
2215         }
2216
2217         // delete from LFIB
2218         //
2219         LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2220
2221         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2222         for ( BigInteger dpId : targetDpns ) {
2223             LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
2224                       vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
2225                       dpId, interVpnLinkName);
2226             makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */,
2227                                LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2228         }
2229         tx.submit();
2230     }
2231 }