git.opendaylight Code Review - netvirt.git/blob

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.util.concurrent.CheckedFuture;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16
  17 import java.math.BigInteger;
  18 import java.net.InetAddress;
  19 import java.net.UnknownHostException;
  20 import java.util.ArrayList;
  21 import java.util.Arrays;
  22 import java.util.Collection;
  23 import java.util.Collections;
  24 import java.util.List;
  25 import java.util.concurrent.BlockingQueue;
  26 import java.util.concurrent.Callable;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30
  31 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  32 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  38 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.mdsalutil.ActionInfo;
  41 import org.opendaylight.genius.mdsalutil.ActionType;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  44 import org.opendaylight.genius.mdsalutil.InstructionType;
  45 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  46 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  47 import org.opendaylight.genius.mdsalutil.MatchInfo;
  48 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  51 import org.opendaylight.genius.utils.ServiceIndex;
  52 import org.opendaylight.genius.utils.batching.ActionableResource;
  53 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  54 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  55 import org.opendaylight.genius.utils.batching.ResourceHandler;
  56 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  57 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  58 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 114 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 115 import org.opendaylight.yangtools.yang.binding.DataObject;
 116 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 117 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 118 import org.opendaylight.yangtools.yang.common.RpcResult;
 119 import org.slf4j.Logger;
 120 import org.slf4j.LoggerFactory;
 121
 122 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> implements AutoCloseable, ResourceHandler {
 123     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 124     private static final String FLOWID_PREFIX = "L3.";
 125     private final DataBroker dataBroker;
 126     private final IMdsalApiManager mdsalManager;
 127     private IVpnManager vpnmanager;
 128     private NexthopManager nextHopManager;
 129     private ItmRpcService itmManager;
 130     private OdlInterfaceRpcService interfaceManager;
 131     private IdManagerService idManager;
 132     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 133     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 134     private static final int LFIB_INTERVPN_PRIORITY = 1;
 135     private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 136     private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 137     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 138     private static final int PERIODICITY = 500;
 139     private static Integer batchSize;
 140     private static Integer batchInterval;
 141     private static final int BATCH_SIZE = 1000;
 142     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 143     private ResourceBatchingManager resourceBatchingManager;
 144
 145     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 146                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 147                             final IdManagerService idManager) {
 148         super(VrfEntry.class, VrfEntryListener.class);
 149         this.dataBroker = dataBroker;
 150         this.mdsalManager = mdsalApiManager;
 151         this.nextHopManager = nexthopManager;
 152         this.interfaceManager = interfaceManager;
 153         this.idManager = idManager;
 154         batchSize = Integer.getInteger("batch.size");
 155         if (batchSize == null) {
 156             batchSize = BATCH_SIZE;
 157         }
 158         batchInterval = Integer.getInteger("batch.wait.time");
 159         if (batchInterval == null) {
 160             batchInterval = PERIODICITY;
 161         }
 162         resourceBatchingManager = ResourceBatchingManager.getInstance();
 163         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 164     }
 165
 166     public void start() {
 167         LOG.info("{} start", getClass().getSimpleName());
 168         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 169     }
 170
 171     @Override
 172     protected VrfEntryListener getDataTreeChangeListener() { return VrfEntryListener.this; }
 173
 174     @Override
 175     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 176         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 177     }
 178
 179     @Override
 180     public DataBroker getResourceBroker() {
 181         return dataBroker;
 182     }
 183
 184     @Override
 185     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 186         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 187         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 188         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 189                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 190         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 191             createFibEntries(identifier, vrfEntry);
 192         } else {
 193             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 194             actResource.setAction(ActionableResource.CREATE);
 195             actResource.setInstanceIdentifier(identifier);
 196             actResource.setInstance(vrfEntry);
 197             vrfEntryBufferQ.add(actResource);
 198             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 199         }
 200         LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 201                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 202     }
 203
 204     @Override
 205     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 206         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 207         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 208         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 209                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 210         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 211             deleteFibEntries(identifier, vrfEntry);
 212         } else {
 213             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 214             actResource.setAction(ActionableResource.DELETE);
 215             actResource.setInstanceIdentifier(identifier);
 216             actResource.setInstance(vrfEntry);
 217             vrfEntryBufferQ.add(actResource);
 218             leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 219         }
 220         LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 221                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 222     }
 223
 224     @Override
 225     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 226         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 227         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 228         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 229                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 230         if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 231             createFibEntries(identifier, update);
 232         } else {
 233             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 234             actResource.setAction(ActionableResource.UPDATE);
 235             actResource.setInstanceIdentifier(identifier);
 236             actResource.setInstance(update);
 237             actResource.setOldInstance(original);
 238             vrfEntryBufferQ.add(actResource);
 239         }
 240         LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 241                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 242     }
 243
 244     @Override
 245     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 246         if (vrfEntry instanceof VrfEntry) {
 247             createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 248         }
 249     }
 250
 251     @Override
 252     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 253         if (vrfEntry instanceof VrfEntry) {
 254             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 255         }
 256     }
 257
 258     @Override
 259     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 260                        Object update) {
 261         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 262             createFibEntries(tx, identifier, (VrfEntry)update);
 263         }
 264     }
 265
 266     @Override
 267     public int getBatchSize() {
 268         return batchSize;
 269     }
 270
 271     @Override
 272     public int getBatchInterval() {
 273         return batchInterval;
 274     }
 275
 276     @Override
 277     public LogicalDatastoreType getDatastoreType() {
 278         return LogicalDatastoreType.CONFIGURATION;
 279     }
 280
 281     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 282         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 283
 284         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 285         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 286         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 287
 288         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 289         final Long vpnId = vpnInstance.getVpnId();
 290         final String rd = vrfTableKey.getRouteDistinguisher();
 291         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 292         if (subnetRoute != null) {
 293             final long elanTag = subnetRoute.getElantag();
 294             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 295                     rd, vrfEntry.getDestPrefix(), elanTag);
 296             if (vpnToDpnList != null) {
 297                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 298                 dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 299                         new Callable<List<ListenableFuture<Void>>>() {
 300                             @Override
 301                             public List<ListenableFuture<Void>> call() throws Exception {
 302                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 303                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
 304                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 305                                         installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 306                                     }
 307                                 }
 308                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
 309                                 futures.add(tx.submit());
 310                                 return futures;
 311                             }
 312                         });
 313             }
 314             return;
 315         }
 316
 317         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 318             // When it is a leaked route, the LFIB and FIB goes a bit different.
 319             installInterVpnRouteInLFib(rd, vrfEntry);
 320             return;
 321         }
 322
 323         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 324
 325         if (vpnToDpnList != null) {
 326             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 327             dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 328                     new Callable<List<ListenableFuture<Void>>>() {
 329                         @Override
 330                         public List<ListenableFuture<Void>> call() throws Exception {
 331                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 332                             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 333                                 if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 334                                     if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 335                                         createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 336                                     }
 337                                 }
 338                             }
 339                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 340                             futures.add(tx.submit());
 341                             return futures;
 342                         }
 343                     });
 344         }
 345
 346         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 347         if ( optVpnUuid.isPresent() ) {
 348             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, optVpnUuid.get());
 349             if ( interVpnLink.isPresent() ) {
 350                 String vpnUuid = optVpnUuid.get();
 351                 String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 352                 if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid, interVpnLink.get()) ) {
 353                     // This is an static route that points to the other endpoint of an InterVpnLink
 354                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 355                     installRouteInInterVpnLink(interVpnLink.get(), vpnUuid, vrfEntry, vpnId);
 356                 }
 357             }
 358         }
 359     }
 360
 361
 362     /*
 363       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 364       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 365       provided by ResourceBatchingManager
 366      */
 367     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 368         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 369
 370         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 371         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 372         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 373
 374         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 375         final String rd = vrfTableKey.getRouteDistinguisher();
 376         if (vpnToDpnList != null) {
 377             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 378                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 379                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 380                 }
 381             }
 382         }
 383     }
 384
 385     /*
 386      * Returns true if the specified nexthop is the other endpoint in an
 387      * InterVpnLink, regarding one of the VPN's point of view.
 388      */
 389     private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) {
 390         return
 391                 interVpnLink != null
 392                         && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
 393                         && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
 394                         || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
 395                         && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
 396     }
 397
 398
 399     // FIXME: Refactoring needed here.
 400     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 401     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 402                                    int addOrRemove) {
 403         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 404         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 405
 406         String rd = vrfTableKey.getRouteDistinguisher();
 407         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 408         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 409             if (vpnInstance == null) {
 410                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 411                 return;
 412             }
 413         } else {
 414             Preconditions.checkNotNull(vpnInstance,
 415                     "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 416         }
 417         String vpnUuid = vpnInstance.getVpnInstanceName();
 418         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 419                 "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 420
 421         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 422         // there an interVpnLink for the involved vpn in order to make learn the new route to
 423         // the other part of the inter-vpn-link.
 424
 425         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 426         Optional<InterVpnLink> interVpnLink =
 427                 (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
 428                         : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
 429         if ( !interVpnLink.isPresent() ) {
 430             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 431             return;
 432         }
 433
 434         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 435         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 436         boolean proceed = (addOrRemove == NwConstants.DEL_FLOW )
 437                 || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 438                 && interVpnLink.get().isBgpRoutesLeaking() );
 439
 440         if ( proceed ) {
 441             String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) )
 442                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 443                     : vpnUuid;
 444
 445             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
 446             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 447
 448             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 449                     InstanceIdentifier.builder(FibEntries.class)
 450                             .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 451                             .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 452                             .build();
 453             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 454                 LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 455                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 456                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 457                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 458                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 459                         .setLabel(label)
 460                         .setOrigin(RouteOrigin.INTERVPN.getValue())
 461                         .build();
 462                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 463             } else {
 464                 LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 465                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 466             }
 467         }
 468     }
 469
 470     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 471         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 472         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 473         prefixBuilder.setDpnId(lri.getDpnId());
 474         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 475         prefixBuilder.setIpAddress(lri.getPrefix());
 476         // Increment the refCount here
 477         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 478                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 479         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 480         if (!isPresentInList) {
 481             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 482             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 483             vpnInstanceNames.add(vpnInstanceName);
 484             builder.setVpnInstanceList(vpnInstanceNames);
 485             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 486         } else {
 487             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 488         }
 489         return prefixBuilder.build();
 490     }
 491
 492     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 493                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 494         Boolean wrTxPresent = true;
 495         if (tx == null) {
 496             wrTxPresent = false;
 497             tx = dataBroker.newWriteOnlyTransaction();
 498         }
 499         synchronized (vrfEntry.getLabel().toString().intern()) {
 500             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 501             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 502                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 503
 504                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 505                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 506                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 507                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 508                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 509                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 510                         }
 511                     }
 512                 }
 513                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 514                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 515             }
 516         }
 517         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 518         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 519         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 520         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 521         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 522
 523         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 524             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 525             // reinitialize instructions list for LFIB Table
 526             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 527
 528             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 529             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 530             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 531             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 532
 533             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 534         }
 535         if (!wrTxPresent ) {
 536             tx.submit();
 537         }
 538     }
 539
 540     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 541         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 542         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 543         // packet is commuted from Vpn2 to Vpn1.
 544         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 545         if ( !vpnNameOpc.isPresent() ) {
 546             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 547             return;
 548         }
 549
 550         String vpnName = vpnNameOpc.get();
 551         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 552         boolean interVpnLinkFound = false;
 553         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 554             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 555             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 556                     && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 557             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 558                 interVpnLinkFound = true;
 559
 560                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 561                 if ( !vpnLinkState.isPresent()
 562                         || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 563                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 564                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 565                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 566                     return;
 567                 }
 568
 569                 List<BigInteger> targetDpns =
 570                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
 571                                 : vpnLinkState.get().getSecondEndpointState().getDpId();
 572                 Long lportTag =
 573                         ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 574                                 : vpnLinkState.get().getFirstEndpointState().getLportTag();
 575
 576                 for ( BigInteger dpId : targetDpns ) {
 577                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 578
 579                     BigInteger[] metadata = new BigInteger[] {
 580                             MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 581                             MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 582                     };
 583                     List<InstructionInfo> instructions =
 584                             Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 585                                     new InstructionInfo(InstructionType.write_metadata, metadata),
 586                                     new InstructionInfo(InstructionType.goto_table,
 587                                             new long[] { NwConstants.L3_INTERFACE_TABLE }));
 588
 589                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 590                             NwConstants.ADD_FLOW, null);
 591                 }
 592
 593                 break;
 594             }
 595         }
 596
 597         if ( !interVpnLinkFound ) {
 598             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 599                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 600         }
 601     }
 602
 603
 604
 605     private void installRouteInInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 606                                             final VrfEntry vrfEntry, long vpnTag) {
 607         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 608         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 609                 && vrfEntry.getNextHopAddressList().size() == 1);
 610         String destination = vrfEntry.getDestPrefix();
 611         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 612
 613         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 614         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 615         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 616         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 617         if ( !interVpnLinkState.isPresent() ) {
 618             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 619             return;
 620         }
 621         if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 622             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 623                     destination, nextHop, interVpnLink.getName());
 624             return;
 625         }
 626
 627
 628         // Everything Ok
 629         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 630         List<BigInteger> targetDpns =
 631                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 632                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 633
 634         Long otherEndpointlportTag =
 635                 vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
 636                         : interVpnLinkState.get().getFirstEndpointState().getLportTag();
 637
 638         BigInteger[] metadata = new BigInteger[] {
 639                 MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag.intValue(),
 640                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
 641                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 642         };
 643         int instIdx = 0;
 644         List<Instruction> instructions =
 645                 Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 646                         new InstructionInfo(InstructionType.goto_table,
 647                                 new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 648
 649         String values[] = destination.split("/");
 650         String destPrefixIpAddress = values[0];
 651         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 652
 653         List<MatchInfo> matches = new ArrayList<>();
 654         matches.add(new MatchInfo(MatchFieldType.metadata,
 655                 new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
 656                         MetaDataUtil.METADATA_MASK_VRFID }));
 657         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 658
 659         if (prefixLength != 0) {
 660             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 661                     new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 662         }
 663
 664         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 665         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), destination, nextHop);
 666         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 667                 COOKIE_VM_FIB_TABLE, matches, instructions);
 668
 669         for ( BigInteger dpId : targetDpns ) {
 670             mdsalManager.installFlow(dpId, flowEntity);
 671         }
 672     }
 673
 674     private void removeRouteFromInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 675                                              final VrfEntry vrfEntry) {
 676
 677         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 678         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 679                 && vrfEntry.getNextHopAddressList().size() == 1);
 680
 681         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 682         if ( !interVpnLinkState.isPresent() ) {
 683             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 684             return;
 685         }
 686
 687         // Everything Ok
 688         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 689         List<BigInteger> targetDpns =
 690                 vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 691                         : interVpnLinkState.get().getSecondEndpointState().getDpId();
 692
 693         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 694         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 695         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
 696         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
 697                 .setFlowName(flowRef).build();
 698
 699         for ( BigInteger dpId : targetDpns ) {
 700             mdsalManager.removeFlow(dpId, flow);
 701         }
 702
 703     }
 704
 705     private boolean isVpnFirstEndPoint(InterVpnLink interVpnLink, String vpnName) {
 706         return interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 707     }
 708
 709     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 710                                                      InstanceIdentifier<T> path) {
 711
 712         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 713
 714         Optional<T> result = Optional.absent();
 715         try {
 716             result = tx.read(datastoreType, path).get();
 717         } catch (Exception e) {
 718             throw new RuntimeException(e);
 719         }
 720
 721         return result;
 722     }
 723
 724     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 725         List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 726         Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 727         String localNextHopIP = vrfEntry.getDestPrefix();
 728
 729         if (localNextHopInfo == null) {
 730             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 731             Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 732             if (extraRoute != null && extraRoute.getNexthopIpList() != null) {
 733                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 734                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 735                     if (nextHopIp != null) {
 736                         localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
 737                         localNextHopIP = nextHopIp + "/32";
 738                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 739                         returnLocalDpnId.add(dpnId);
 740                     }
 741                 }
 742             }
 743             if (localNextHopInfo == null) {
 744             /* imported routes case */
 745                 synchronized (vrfEntry.getLabel().toString().intern()) {
 746                     LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 747                     if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 748                             vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 749                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 750                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
 751                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
 752                                 String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 753                                 if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 754                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 755                                     localNextHopIP = lri.getPrefix();
 756                                 } else {
 757                                     localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 758                                     localNextHopIP = lri.getPrefix();
 759                                 }
 760                             }
 761                             if (localNextHopInfo != null) {
 762                                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 763                                         vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 764                                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 765                                 returnLocalDpnId.add(dpnId);
 766                             }
 767                         }
 768                     }
 769                 }
 770             }
 771         } else {
 772             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 773             returnLocalDpnId.add(dpnId);
 774         }
 775
 776         return returnLocalDpnId;
 777     }
 778
 779     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 780                                                 final VrfEntry vrfEntry, Long parentVpnId){
 781         if (localNextHopInfo != null) {
 782             final BigInteger dpnId = localNextHopInfo.getDpnId();
 783             if (!isVpnPresentInDpn(rd, dpnId)) {
 784                 return BigInteger.ZERO;
 785             }
 786
 787             final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
 788
 789             List<ActionInfo> actionsInfos =
 790                     Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 791             final List<InstructionInfo> instructions =
 792                     Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 793             actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 794                     new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 795             final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 796             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 797                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 798                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 799             } else {
 800                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 801             }
 802             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 803             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 804                     new Callable<List<ListenableFuture<Void>>>() {
 805                         @Override
 806                         public List<ListenableFuture<Void>> call() throws Exception {
 807                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 808                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 809                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 810                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 811                                 makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 812                             }
 813                             List<ListenableFuture<Void>> futures = new ArrayList<>();
 814                             futures.add(tx.submit());
 815                             return futures;
 816                         }
 817                     });
 818             return dpnId;
 819         }
 820         return BigInteger.ZERO;
 821     }
 822
 823     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 824         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 825         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 826         if (dpnInVpn.isPresent()) {
 827             return true;
 828         }
 829         return false;
 830     }
 831
 832     private LabelRouteInfo getLabelRouteInfo(Long label) {
 833         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 834                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 835         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 836         if (opResult.isPresent()) {
 837             return opResult.get();
 838         }
 839         return null;
 840     }
 841
 842     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 843         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 844         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 845                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 846         if (lri == null) {
 847             return true;
 848         }
 849         List<String> vpnInstancesList = lri.getVpnInstanceList();
 850         if (vpnInstancesList.contains(vpnInstanceName)) {
 851             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 852             vpnInstancesList.remove(vpnInstanceName);
 853         }
 854         if (vpnInstancesList.size() == 0) {
 855             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 856             FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 857             return true;
 858         } else {
 859             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 860             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 861             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 862         }
 863         return false;
 864     }
 865
 866     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 867                                       WriteTransaction tx) {
 868         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 869         actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 870
 871
 872         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 873
 874         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 875                 dpId, label, groupId);
 876     }
 877
 878     public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 879                                                  WriteTransaction tx) {
 880         List<MatchInfo> mkMatches = new ArrayList<>();
 881
 882         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 883
 884         // Matching metadata
 885         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 886         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 887
 888         List<InstructionInfo> mkInstructions = new ArrayList<>();
 889         mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 890
 891         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 892                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 893                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 894
 895         FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 896
 897         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 898
 899         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 900         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 901                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 902                 .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 903         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 904     }
 905
 906     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 907         FlowEntity flowEntity;
 908         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 909         List<MatchInfo> mkMatches = new ArrayList<>();
 910         // Matching metadata
 911         mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 912         flowEntity = MDSALUtil.buildFlowEntity(dpId,
 913                 NwConstants.INTERNAL_TUNNEL_TABLE,
 914                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 915                 5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 916                 COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 917         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 918         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 919         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 920                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 921                 .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 922
 923         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 924         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 925     }
 926
 927     /**
 928      * Delete local FIB entry
 929      * @param vpnId
 930      * @param rd
 931      * @param vrfEntry
 932      * @return
 933      */
 934     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 935         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 936         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 937         String localNextHopIP = vrfEntry.getDestPrefix();
 938
 939         if (localNextHopInfo == null) {
 940             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 941             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 942             if (extra_route != null) {
 943                 for (String nextHopIp : extra_route.getNexthopIpList()) {
 944                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 945                     if (nextHopIp != null) {
 946                         localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 947                         localNextHopIP = nextHopIp + "/32";
 948                         BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 949                                 vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 950                         if (!dpnId.equals(BigInteger.ZERO)) {
 951                             returnLocalDpnId.add(dpnId);
 952                         }
 953                     }
 954                 }
 955             }
 956
 957             if (localNextHopInfo == null) {
 958               /* Imported VRF entry */
 959                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 960                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 961                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 962                     VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
 963                     vpnNexthopBuilder.setDpnId(lri.getDpnId());
 964                     BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
 965                             vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 966                     if (!dpnId.equals(BigInteger.ZERO)) {
 967                         returnLocalDpnId.add(dpnId);
 968                     }
 969                 }
 970             }
 971
 972
 973         } else {
 974             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 975                     vpnId, rd, vrfEntry, false /*isExtraRoute*/);
 976             if (!dpnId.equals(BigInteger.ZERO)) {
 977                 returnLocalDpnId.add(dpnId);
 978             }
 979         }
 980
 981         return returnLocalDpnId;
 982     }
 983
 984     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
 985                                                 final Long vpnId, final String rd,
 986                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
 987         if (localNextHopInfo != null) {
 988             final BigInteger dpnId = localNextHopInfo.getDpnId();;
 989             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 990             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 991                     new Callable<List<ListenableFuture<Void>>>() {
 992                         @Override
 993                         public List<ListenableFuture<Void>> call() throws Exception {
 994                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 995                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
 996                                     NwConstants.DEL_FLOW, tx);
 997                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 998                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
 999                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1000                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1001                             }
1002                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1003                             futures.add(tx.submit());
1004                             return futures;
1005                         }
1006                     });
1007             //TODO: verify below adjacency call need to be optimized (?)
1008             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
1009             return dpnId;
1010         }
1011         return BigInteger.ZERO;
1012     }
1013
1014
1015     private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
1016         return InstanceIdentifier.builder(PrefixToInterface.class)
1017                 .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
1018     }
1019
1020     private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
1021         Optional<Prefixes> localNextHopInfoData =
1022                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
1023         return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
1024     }
1025
1026     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1027         return InstanceIdentifier.builder(VpnToExtraroute.class)
1028                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1029                         new ExtrarouteKey(ipPrefix)).build();
1030     }
1031
1032     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1033         Optional<Extraroute> extraRouteInfo =
1034                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1035         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1036
1037     }
1038
1039     private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1040         try {
1041             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1042                     new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1043             RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1044             if(!rpcResult.isSuccessful()) {
1045                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1046             } else {
1047                 return rpcResult.getResult().getTunnelType();
1048             }
1049
1050         } catch (InterruptedException | ExecutionException e) {
1051             LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
1052         }
1053
1054         return null;
1055
1056     }
1057     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1058                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1059         Boolean wrTxPresent = true;
1060         if (tx == null) {
1061             wrTxPresent = false;
1062             tx = dataBroker.newWriteOnlyTransaction();
1063         }
1064         String rd = vrfTableKey.getRouteDistinguisher();
1065         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1066                 vrfEntry.getDestPrefix(), rd, tx);
1067         /********************************************/
1068         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1069
1070         if (tunnelInterfaceList.isEmpty()) {
1071             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1072                     vrfEntry.getNextHopAddressList(), rd);
1073             LOG.warn("Failed to add Route: {} in vpn: {}",
1074                     vrfEntry.getDestPrefix(), rd);
1075             return;
1076         }
1077
1078         for (String tunnelInterface : tunnelInterfaceList) {
1079             List<InstructionInfo> instructions = new ArrayList<>();
1080             List<ActionInfo> actionInfos = new ArrayList<>();
1081             Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1082             if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1083                 LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1084                 actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
1085                 actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
1086             } else {
1087                 int label = vrfEntry.getLabel().intValue();
1088                 BigInteger tunnelId;
1089                 // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1090                 if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1091                     tunnelId = BigInteger.valueOf(label);
1092                 } else {
1093                     tunnelId = BigInteger.valueOf(label);
1094                 }
1095
1096                 LOG.debug("adding set tunnel id action for label {}", label);
1097                 actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
1098             }
1099             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
1100             if(egressActions.isEmpty()){
1101                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
1102                 return;
1103             }
1104             actionInfos.addAll(egressActions);
1105             instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
1106             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1107         }
1108         if(!wrTxPresent ){
1109             tx.submit();
1110         }
1111         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1112     }
1113
1114     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1115         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1116         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1117         if (dpnInVpn.isPresent()) {
1118             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1119             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1120
1121             if (vpnInterfaces.remove(currVpnInterface)) {
1122                 if (vpnInterfaces.isEmpty()) {
1123                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1124                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1125                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1126                 } else {
1127                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1128                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1129                             VpnInterfaces.class,
1130                             new VpnInterfacesKey(intfName)));
1131                 }
1132             }
1133         }
1134     }
1135
1136     private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1137     /* Get interface info from prefix to interface mapping;
1138         Use the interface info to get the corresponding vpn interface op DS entry,
1139         remove the adjacency corresponding to this fib entry.
1140         If adjacency removed is the last adjacency, clean up the following:
1141          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1142          - prefix to interface entry
1143          - vpn interface op DS
1144      */
1145         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1146         Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1147         Extraroute extraRoute = null;
1148         if (prefixInfo == null) {
1149             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1150             if(extraRoute != null) {
1151                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1152                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1153
1154                     if (nextHopIp != null) {
1155                         prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
1156                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1157                     }
1158                 }
1159             }
1160             if (prefixInfo == null) {
1161                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1162                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1163                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1164                     PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1165                     prefixBuilder.setDpnId(lri.getDpnId());
1166                     prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1167                     prefixBuilder.setIpAddress(lri.getPrefix());
1168                     prefixInfo = prefixBuilder.build();
1169                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1170                             vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1171                     checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1172                 }
1173             }
1174         } else {
1175             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1176         }
1177     }
1178
1179     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1180                                           final VrfEntry vrfEntry, final Extraroute extraRoute) {
1181
1182         if (prefixInfo == null) {
1183             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1184             return; //Don't have any info for this prefix (shouldn't happen); need to return
1185         }
1186
1187         String ifName = prefixInfo.getVpnInterfaceName();
1188         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1189         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1190                 new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1191     }
1192
1193     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1194         Prefixes prefixInfo;
1195         Long vpnId;
1196         String rd;
1197         VrfEntry vrfEntry;
1198         Extraroute extraRoute;
1199
1200         public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1201                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
1202             this.prefixInfo = prefixInfo;
1203             this.vpnId = vpnId;
1204             this.rd= rd;
1205             this.vrfEntry= vrfEntry;
1206             this.extraRoute = extraRoute;
1207         }
1208
1209         @Override
1210         public List<ListenableFuture<Void>> call() throws Exception {
1211             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1212             // to call the respective helpers.
1213             String ifName = prefixInfo.getVpnInterfaceName();
1214             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1215                     FibUtil.getVpnInterfaceIdentifier(ifName));
1216             if (optvpnInterface.isPresent()) {
1217                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1218                 if (vpnId != associatedVpnId) {
1219                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1220                             vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1221                     LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
1222                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1223                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1224                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1225                     return null;
1226                 } else {
1227                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1228                             vrfEntry.getDestPrefix(), associatedVpnId);
1229                 }
1230             }
1231             if (extraRoute != null) {
1232                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1233                         FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1234             }
1235             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1236                     FibUtil.getAdjListPath(ifName));
1237             int numAdj = 0;
1238             if (optAdjacencies.isPresent()) {
1239                 numAdj = optAdjacencies.get().getAdjacency().size();
1240             }
1241             //remove adjacency corr to prefix
1242             if (numAdj > 1) {
1243                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1244                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1245                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1246             }
1247             if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1248                 //clean up the vpn interface from DpnToVpn list
1249                 LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1250                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
1251                         FibUtil.getVpnInterfaceIdentifier(ifName));
1252             }
1253
1254             synchronized (vrfEntry.getLabel().toString().intern()) {
1255                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1256                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1257                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1258                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1259                     String vpnInstanceName = "";
1260                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1261                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1262                     }
1263                     boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1264                     if (lriRemoved) {
1265                         String parentRd = lri.getParentVpnRd();
1266                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1267                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1268                     }
1269                 } else {
1270                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1271                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1272                 }
1273             }
1274             return null;
1275         }
1276     }
1277
1278     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1279         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1280         final String rd  = vrfTableKey.getRouteDistinguisher();
1281         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1282         if (vpnInstance == null) {
1283             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1284             return;
1285         }
1286         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1287         long elanTag = 0L;
1288         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1289         if (subnetRoute != null) {
1290             elanTag = subnetRoute.getElantag();
1291             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1292                     rd, vrfEntry.getDestPrefix(), elanTag);
1293             if (vpnToDpnList != null) {
1294                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1295                 dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1296                         new Callable<List<ListenableFuture<Void>>>() {
1297                             @Override
1298                             public List<ListenableFuture<Void>> call() throws Exception {
1299                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1300
1301                                 for (final VpnToDpnList curDpn : vpnToDpnList) {
1302
1303                                     makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1304                                             vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1305                                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1306                                         makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1307                                                 DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1308                                     }
1309                                 }
1310                                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1311                                 futures.add(tx.submit());
1312                                 return futures;
1313                             }
1314                         });
1315             }
1316             synchronized (vrfEntry.getLabel().toString().intern()) {
1317                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1318                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1319                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
1320                     String vpnInstanceName = "";
1321                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
1322                         vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1323                     }
1324                     boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1325                     if (lriRemoved) {
1326                         String parentRd = lri.getParentVpnRd();
1327                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1328                                 FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1329                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
1330                                 vrfEntry.getDestPrefix());
1331                     }
1332                 } else {
1333                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1334                             FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1335                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1336                             vrfEntry.getDestPrefix());
1337                 }
1338             }
1339             return;
1340         }
1341
1342         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1343                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
1344         if (vpnToDpnList != null) {
1345             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1346             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1347                     new Callable<List<ListenableFuture<Void>>>() {
1348                         @Override
1349                         public List<ListenableFuture<Void>> call() throws Exception {
1350                             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1351
1352                             if (localDpnIdList.size() <= 0) {
1353                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1354                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1355                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1356                                             deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1357                                         }
1358                                     } else {
1359                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1360                                     }
1361                                 }
1362                             } else {
1363                                 for (BigInteger localDpnId : localDpnIdList) {
1364                                     for (VpnToDpnList curDpn : vpnToDpnList) {
1365                                         if (!curDpn.getDpnId().equals(localDpnId)) {
1366                                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1367                                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1368                                                     deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1369                                                 }
1370                                             } else {
1371                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1372                                             }
1373                                         }
1374                                     }
1375                                 }
1376                             }
1377                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1378                             futures.add(tx.submit());
1379                             return futures;
1380                         }
1381                     });
1382         }
1383
1384         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1385         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1386         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1387
1388         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1389         // of the interVpnLink.
1390         Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
1391         if ( vpnUuid.isPresent() ) {
1392             Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, vpnUuid.get());
1393             String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
1394
1395             if ( interVpnLink.isPresent()
1396                     && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
1397                     && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
1398                     || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
1399                     && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
1400                 // This is route that points to the other endpoint of an InterVpnLink
1401                 // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1402                 removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
1403             }
1404         }
1405
1406     }
1407
1408     /*
1409       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1410       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1411       provided by ResourceBatchingManager
1412      */
1413     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1414         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1415
1416         final String rd  = vrfTableKey.getRouteDistinguisher();
1417         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1418         if (vpnInstance == null) {
1419             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1420             return;
1421         }
1422         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1423         if (vpnToDpnList != null) {
1424             for (VpnToDpnList curDpn : vpnToDpnList) {
1425                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1426                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1427                 }
1428             }
1429         }
1430     }
1431
1432     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1433                                   final long vpnId, final VrfTablesKey vrfTableKey,
1434                                   final VrfEntry vrfEntry, WriteTransaction tx) {
1435
1436         Boolean wrTxPresent = true;
1437         if (tx == null) {
1438             wrTxPresent = false;
1439             tx = dataBroker.newWriteOnlyTransaction();
1440         }
1441
1442         LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1443         String rd = vrfTableKey.getRouteDistinguisher();
1444
1445         if(localDpnId != null) {
1446             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1447             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1448             return;
1449         }
1450
1451         // below two reads are kept as is, until best way is found to identify dpnID
1452         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1453         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1454
1455         if (localNextHopInfo == null && extraRoute != null) {
1456             // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1457             for (String nextHopIp : extraRoute.getNexthopIpList()) {
1458                 localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1459                 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1460             }
1461         } else {
1462             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1463         }
1464         if(!wrTxPresent ){
1465             tx.submit();
1466         }
1467     }
1468
1469     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1470                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1471         boolean isRemoteRoute = true;
1472         if (localNextHopInfo != null) {
1473             isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId()));
1474         }
1475         if (isRemoteRoute) {
1476             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1477             return true;
1478         } else {
1479             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1480             return false;
1481         }
1482     }
1483
1484     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1485         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1486         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1487     }
1488
1489     private long get
1490             (byte[] rawIpAddress) {
1491         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1492                 + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1493     }
1494
1495     private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1496                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1497         Boolean wrTxPresent = true;
1498         if (tx == null) {
1499             wrTxPresent = false;
1500             tx = dataBroker.newWriteOnlyTransaction();
1501         }
1502
1503         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1504         String values[] = vrfEntry.getDestPrefix().split("/");
1505         String ipAddress = values[0];
1506         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1507         if (addOrRemove == NwConstants.ADD_FLOW) {
1508             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1509         } else {
1510             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1511         }
1512         InetAddress destPrefix;
1513         try {
1514             destPrefix = InetAddress.getByName(ipAddress);
1515         } catch (UnknownHostException e) {
1516             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1517             return;
1518         }
1519
1520         List<MatchInfo> matches = new ArrayList<>();
1521
1522         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1523                 MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1524
1525         matches.add(new MatchInfo(MatchFieldType.eth_type,
1526                 new long[] { NwConstants.ETHTYPE_IPV4 }));
1527
1528         if(prefixLength != 0) {
1529             matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1530                     destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1531         }
1532         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1533         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1534         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1535                 COOKIE_VM_FIB_TABLE, matches, instructions);
1536
1537         Flow flow = flowEntity.getFlowBuilder().build();
1538         String flowId = flowEntity.getFlowId();
1539         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1540         Node nodeDpn = buildDpnNode(dpId);
1541
1542         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1543                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1544                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1545         if (addOrRemove == NwConstants.ADD_FLOW) {
1546             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1547         } else {
1548             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1549         }
1550
1551         if(!wrTxPresent ){
1552             tx.submit();
1553         }
1554     }
1555
1556     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1557     private Node buildDpnNode(BigInteger dpnId) {
1558         NodeId nodeId = new NodeId("openflow:" + dpnId);
1559         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1560
1561         return nodeDpn;
1562     }
1563
1564     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1565                                     int addOrRemove, WriteTransaction tx) {
1566         Boolean wrTxPresent = true;
1567         if (tx == null) {
1568             wrTxPresent = false;
1569             tx = dataBroker.newWriteOnlyTransaction();
1570         }
1571
1572         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1573         matches.add(new MatchInfo(MatchFieldType.eth_type,
1574                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1575         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1576
1577         // Install the flow entry in L3_LFIB_TABLE
1578         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1579
1580         FlowEntity flowEntity;
1581         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1582                 NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1583         Flow flow = flowEntity.getFlowBuilder().build();
1584         String flowId = flowEntity.getFlowId();
1585         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1586         Node nodeDpn = buildDpnNode(dpId);
1587         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1588                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1589                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1590
1591         if (addOrRemove == NwConstants.ADD_FLOW) {
1592             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1593         } else {
1594             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1595         }
1596         if(!wrTxPresent ){
1597             tx.submit();
1598         }
1599         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
1600                 dpId, label, instructions );
1601     }
1602
1603     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
1604         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1605         try {
1606             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
1607         } catch (NullPointerException e) {
1608             LOG.trace("", e);
1609         }
1610     }
1611
1612     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1613             final FutureCallback<List<Void>> callback) {
1614         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1615         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1616         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1617         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1618         if (!vrfTable.isPresent()) {
1619             LOG.warn("VRF Table not yet available for RD {}", rd);
1620             if (callback != null) {
1621                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1622                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1623                 Futures.addCallback(listenableFuture, callback);
1624             }
1625             return;
1626         }
1627         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1628         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
1629                 new Callable<List<ListenableFuture<Void>>>() {
1630             @Override
1631             public List<ListenableFuture<Void>> call() throws Exception {
1632                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1633                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1634                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1635                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1636                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1637                         if (subnetRoute != null) {
1638                             long elanTag = subnetRoute.getElantag();
1639                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1640                             continue;
1641                         }
1642                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
1643                             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1644                             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1645                                     && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1646                                 if (lri.getDpnId().equals(dpnId)) {
1647                                     createLocalFibEntry(vpnId, rd, vrfEntry);
1648                                     continue;
1649                                 }
1650                             }
1651                         }
1652                         // Passing null as we don't know the dpn
1653                         // to which prefix is attached at this point
1654                         createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1655                     }
1656                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1657                     futures.add(tx.submit());
1658                 }
1659                 if (callback != null) {
1660                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1661                     Futures.addCallback(listenableFuture, callback);
1662                 }
1663                 return futures;
1664             }
1665         });
1666     }
1667
1668
1669     public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1670                                  final String localNextHopIp, final String remoteNextHopIp) {
1671         LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1672                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1673         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1674         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1675         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1676         if (vrfTable.isPresent()) {
1677             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1678             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1679                     new Callable<List<ListenableFuture<Void>>>() {
1680                         @Override
1681                         public List<ListenableFuture<Void>> call() throws Exception {
1682                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1683                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1684                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1685                                 LOG.trace("populate FIB starts on Dpn " + dpnId
1686                                         + "rd  " + rd.toString()
1687                                         + "localNextHopIp " + localNextHopIp
1688                                         + "remoteNextHopIp" + remoteNextHopIp
1689                                         + "vpnId " + vpnId );
1690                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1691                                     LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1692
1693                                     if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1694                                         if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1695                                             LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1696                                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1697                                         }
1698                                     } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
1699                                         BigInteger dpnIdForPrefix = null;
1700                                         String destPfx = vrfEntry.getDestPrefix();
1701                                         if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
1702                                             Optional<Extraroute> extraRouteInfo =
1703                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1704                                                             getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1705                                             if (extraRouteInfo.isPresent()) {
1706                                                 continue;
1707                                             }
1708                                             dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1709                                         } else {
1710                                             // Subnet Route handling
1711                                             Optional<Prefixes> localNextHopInfoData =
1712                                                     FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1713                                                             FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
1714                                             if (localNextHopInfoData.isPresent()) {
1715                                                 Prefixes prefixes = localNextHopInfoData.get();
1716                                                 dpnIdForPrefix = prefixes.getDpnId();
1717                                             }
1718                                         }
1719                                         if (dpnIdForPrefix == null) {
1720                                             LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1721                                                     vrfEntry.getDestPrefix());
1722                                             continue;
1723                                         }
1724                                         int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1725                                         if (sameDpnId != 0) {
1726                                             LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1727                                                     dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1728                                             continue;
1729                                         }
1730                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1731                                         List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
1732                                         newNextHopAddrList.add(localNextHopIp);
1733                                         VrfEntry newVrfEntry =
1734                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
1735                                         // Just update the VrfEntry
1736                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1737                                                 vrfEntryId, newVrfEntry);
1738                                         // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
1739                                         //       vrfEntryId, newVrfEntry);
1740                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1741                                         LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1742                                     }
1743                                 }
1744                                 futures.add(writeTransaction.submit());
1745                                 LOG.trace("populate FIB ends on Dpn " + dpnId
1746                                         + "rd  " + rd.toString()
1747                                         + "localNextHopIp " + localNextHopIp
1748                                         + "remoteNextHopIp" + remoteNextHopIp
1749                                         + "vpnId " + vpnId);
1750                             }
1751                             return futures;
1752                         }
1753                     });
1754         }
1755     }
1756
1757     public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
1758                                   final long vpnId, final String  rd, final String destPrefix ,
1759                                   final String localNextHopIP, final String remoteNextHopIp) {
1760
1761         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1762         dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1763                         + "local dpid" + localDpnId
1764                         + "remote dpid" + remoteDpnId
1765                         + "vpnId" + vpnId
1766                         + "localNHIp" + localNextHopIP
1767                         + "remoteNHIp" + remoteNextHopIp,
1768                 new Callable<List<ListenableFuture<Void>>>() {
1769                     @Override
1770                     public List<ListenableFuture<Void>> call() throws Exception {
1771                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1772                         WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1773                         VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1774                         VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1775                         if (vrfEntry == null)
1776                             return futures;
1777                         LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1778                                         "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1779                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1780                         if (action == true) {
1781                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1782                             LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1783                             createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1784                         } else {
1785                             vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
1786                             LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1787                             deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1788                         }
1789                         futures.add(writeTransaction.submit());
1790                         return futures;
1791                     }
1792                 });
1793     }
1794
1795     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1796                                  final FutureCallback<List<Void>> callback) {
1797         LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1798         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1799         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1800         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1801         if (vrfTable.isPresent()) {
1802             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1803             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1804                     new Callable<List<ListenableFuture<Void>>>() {
1805                         @Override
1806                         public List<ListenableFuture<Void>> call() throws Exception {
1807                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1808                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1809                                 WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1810                                 for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1811                                     /* Handle subnet routes here */
1812                                     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1813                                     if (subnetRoute != null) {
1814                                         LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1815                                                 dpnId, rd);
1816                                         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1817                                         makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1818                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1819                                                 vrfEntry.getDestPrefix());
1820                                         continue;
1821                                     }
1822                                     // Passing null as we don't know the dpn
1823                                     // to which prefix is attached at this point
1824                                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1825                                 }
1826                                 futures.add(tx.submit());
1827                                 if (callback != null) {
1828                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1829                                     Futures.addCallback(listenableFuture, callback);
1830                                 }
1831                             }
1832                             return futures;
1833                         }
1834
1835                     });
1836         }
1837     }
1838
1839     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1840                                  final String localNextHopIp, final String remoteNextHopIp,
1841                                  final FutureCallback<List<Void>> callback) {
1842         LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1843                         " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1844                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1845         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1846         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1847         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1848         if (vrfTable.isPresent()) {
1849             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1850             dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
1851                     new Callable<List<ListenableFuture<Void>>>() {
1852                         @Override
1853                         public List<ListenableFuture<Void>> call() throws Exception {
1854                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1855                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
1856                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
1857                                 LOG.trace("cleanup FIB starts on Dpn " + dpnId
1858                                         + "rd  " + rd.toString()
1859                                         + "localNextHopIp " + localNextHopIp
1860                                         + "remoteNextHopIp" + remoteNextHopIp
1861                                         + "vpnId " + vpnId);
1862
1863                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1864                                     LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1865                                     if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1866                                         LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1867                                         deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1868                                     }
1869
1870                                     if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1871                                         LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1872                                                 vrfEntry.getDestPrefix(), dpnId);
1873                                         String destPfx = vrfEntry.getDestPrefix();
1874                                         InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1875                                         List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1876                                         newList.remove(localNextHopIp);
1877                                         VrfEntry newVrfEntry =
1878                                                 new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1879                                         FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
1880                                                 vrfEntryId, newVrfEntry);
1881                                         vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
1882                                         LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1883                                     }
1884                                 }
1885                                 futures.add(writeTransaction.submit());
1886                                 if (callback != null) {
1887                                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1888                                     Futures.addCallback(listenableFuture, callback);
1889                                 }
1890                                 LOG.trace("cleanup FIB ends on Dpn " + dpnId
1891                                         + "rd  " + rd.toString()
1892                                         + "localNextHopIp " + localNextHopIp
1893                                         + "remoteNextHopIp" + remoteNextHopIp
1894                                         + "vpnId " + vpnId);
1895                             }
1896                             return futures;
1897                         }
1898                     });
1899
1900         }
1901     }
1902
1903     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1904         InstanceIdentifierBuilder<VrfTables> idBuilder =
1905                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1906         InstanceIdentifier<VrfTables> id = idBuilder.build();
1907         return id;
1908     }
1909
1910     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1911         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1912                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1913                 .append(priority).toString();
1914     }
1915
1916     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1917         return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1918                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1919                 .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1920                 .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1921                 .append(destPrefix.getHostAddress()).toString();
1922     }
1923
1924     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1925         return new StringBuilder(64).append(FLOWID_PREFIX)
1926                 .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1927                 .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1928                 .append(nextHop).toString();
1929     }
1930
1931     protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
1932                                             final VrfEntry vrfEntry, String rd) {
1933         List<String> adjacencyList = new ArrayList<>();
1934         List<String> prefixIpList = new ArrayList<>();
1935         LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
1936                 remoteDpnId, vpnId, vrfEntry);
1937         try {
1938             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1939                 Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1940                 if (extra_route == null) {
1941                     prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1942                 } else {
1943                     prefixIpList = new ArrayList<>();
1944                     if (extra_route.getNexthopIpList() != null) {
1945                         for (String extraRouteIp : extra_route.getNexthopIpList()) {
1946                             prefixIpList.add(extraRouteIp + "/32");
1947                         }
1948                     }
1949                 }
1950             } else {
1951                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1952             }
1953
1954             for (String prefixIp : prefixIpList) {
1955                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
1956                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
1957                     String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
1958                             prefixIp, nextHopIp);
1959                     if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
1960                         adjacencyList.add(adjacency);
1961                     }
1962                 }
1963             }
1964         } catch (NullPointerException e) {
1965             LOG.trace("", e);
1966         }
1967         return adjacencyList;
1968     }
1969
1970     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
1971         InstanceIdentifier<VpnInstanceOpDataEntry> id =
1972                 InstanceIdentifier.create(VpnInstanceOpData.class)
1973                         .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
1974         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
1975                 FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1976         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
1977     }
1978
1979     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
1980         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1981                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
1982                 .append(FLOWID_PREFIX).toString();
1983     }
1984
1985     /*
1986      * Install flow entry in protocol table to forward mpls
1987      * coming through gre tunnel to LFIB table.
1988      */
1989     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
1990         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
1991         // Instruction to goto L3 InterfaceTable
1992         List<InstructionInfo> instructions = new ArrayList<>();
1993         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
1994         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1995         matches.add(new MatchInfo(MatchFieldType.eth_type,
1996                 new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1997         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
1998                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
1999                         NwConstants.L3_LFIB_TABLE),
2000                 DEFAULT_FIB_FLOW_PRIORITY,
2001                 "Protocol Table For LFIB",
2002                 0, 0,
2003                 COOKIE_PROTOCOL_TABLE,
2004                 matches, instructions);
2005
2006         if (addOrRemove == NwConstants.ADD_FLOW) {
2007             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2008             mdsalManager.installFlow(flowEntityToLfib);
2009         } else {
2010             mdsalManager.removeFlow(flowEntityToLfib);
2011         }
2012     }
2013
2014     public List<String> printFibEntries() {
2015         List<String> result = new ArrayList<String>();
2016         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2017         result.add("-------------------------------------------------------------------");
2018         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2019         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2020         if (fibEntries.isPresent()) {
2021             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2022             for (VrfTables vrfTable : vrfTables) {
2023                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2024                     for (String nextHop : vrfEntry.getNextHopAddressList()) {
2025                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2026                                 vrfTable.getRouteDistinguisher(),
2027                                 vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2028                     }
2029                     if (vrfEntry.getNextHopAddressList().isEmpty()) {
2030                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2031                                 vrfTable.getRouteDistinguisher(),
2032                                 vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2033                     }
2034                 }
2035             }
2036         }
2037         return result;
2038     }
2039
2040
2041     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2042         InstanceIdentifier<VrfEntry> vrfEntryId =
2043                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2044                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2045         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2046         if (vrfEntry.isPresent())  {
2047             return (vrfEntry.get());
2048         }
2049         return null;
2050     }
2051
2052     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2053         InstanceIdentifier<VrfEntry> vrfEntryId =
2054                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2055                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2056         return vrfEntryId;
2057     }
2058 }