vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManager.java

   1 /*
   2  * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12
  13 import java.math.BigInteger;
  14 import java.net.InetAddress;
  15 import java.net.UnknownHostException;
  16 import java.util.Collection;
  17 import java.util.List;
  18 import java.util.concurrent.*;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21
  22 import com.google.common.util.concurrent.CheckedFuture;
  23 import com.google.common.util.concurrent.ListenableFuture;
  24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  25 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
  26 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  27 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  28 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
  29 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  30 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  31 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  32 import org.opendaylight.genius.utils.batching.ActionableResource;
  33 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  34 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  35 import org.opendaylight.genius.utils.batching.ResourceHandler;
  36 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  37 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  38 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
  39 import org.opendaylight.genius.mdsalutil.ActionInfo;
  40 import org.opendaylight.genius.mdsalutil.ActionType;
  41 import org.opendaylight.genius.mdsalutil.FlowEntity;
  42 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  43 import org.opendaylight.genius.mdsalutil.InstructionType;
  44 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  45 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  46 import org.opendaylight.genius.mdsalutil.MatchInfo;
  47 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  50 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  83
  84
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn
  89         .link.states.InterVpnLinkState;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn
  91         .links.InterVpnLink;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 110 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 111 import org.opendaylight.yangtools.yang.binding.DataObject;
 112 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 113 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 114 import org.opendaylight.yangtools.yang.common.RpcResult;
 115 import org.slf4j.Logger;
 116 import org.slf4j.LoggerFactory;
 117
 118 public class FibManager extends AbstractDataChangeListener<VrfEntry> implements AutoCloseable, ResourceHandler {
 119   private static final Logger LOG = LoggerFactory.getLogger(FibManager.class);
 120   private static final String FLOWID_PREFIX = "L3.";
 121   private ListenerRegistration<DataChangeListener> listenerRegistration;
 122   private final DataBroker broker;
 123   private IMdsalApiManager mdsalManager;
 124   private IVpnManager vpnmanager;
 125   private NexthopManager nextHopManager;
 126   private ItmRpcService itmManager;
 127
 128   private OdlInterfaceRpcService interfaceManager;
 129   private IdManagerService idManager;
 130   private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000002", 16);
 131   private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 132   private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 133   private static final int LFIB_INTERVPN_PRIORITY = 1;
 134   private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
 135   private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
 136   public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 137
 138     private static final int PERIODICITY = 500;
 139     private static Integer batchSize;
 140     private static Integer batchInterval;
 141
 142     private static final int BATCH_SIZE = 1000;
 143
 144     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 145     private ResourceBatchingManager resourceBatchingManager;
 146
 147
 148   public FibManager(final DataBroker db) {
 149       super(VrfEntry.class);
 150       broker = db;
 151       registerListener(db);
 152       batchSize = Integer.getInteger("batch.size");
 153       if (batchSize == null) {
 154           batchSize = BATCH_SIZE;
 155       }
 156       batchInterval = Integer.getInteger("batch.wait.time");
 157       if (batchInterval == null) {
 158           batchInterval = PERIODICITY;
 159       }
 160       resourceBatchingManager = ResourceBatchingManager.getInstance();
 161       resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
 162   }
 163
 164   @Override
 165   public void close() throws Exception {
 166     if (listenerRegistration != null) {
 167       try {
 168         listenerRegistration.close();
 169       } catch (final Exception e) {
 170         LOG.error("Error when cleaning up DataChangeListener.", e);
 171       }
 172       listenerRegistration = null;
 173     }
 174     LOG.info("Fib Manager Closed");
 175   }
 176
 177   public void setNextHopManager(NexthopManager nextHopManager) {
 178     this.nextHopManager = nextHopManager;
 179   }
 180     public NexthopManager getNextHopManager() {
 181         return this.nextHopManager;
 182     }
 183
 184   public void setMdsalManager(IMdsalApiManager mdsalManager) {
 185     this.mdsalManager = mdsalManager;
 186   }
 187
 188   public void setVpnmanager(IVpnManager vpnmanager) {
 189     this.vpnmanager = vpnmanager;
 190   }
 191
 192   public void setITMRpcService(ItmRpcService itmManager) {
 193       this.itmManager = itmManager;
 194   }
 195
 196   public void setInterfaceManager(OdlInterfaceRpcService ifManager) {
 197       this.interfaceManager = ifManager;
 198   }
 199
 200   public void setIdManager(IdManagerService idManager) {
 201         this.idManager = idManager;
 202   }
 203
 204   private void registerListener(final DataBroker db) {
 205     try {
 206       listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
 207                                                            getWildCardPath(), FibManager.this, DataChangeScope.SUBTREE);
 208     } catch (final Exception e) {
 209       LOG.error("FibManager DataChange listener registration fail!", e);
 210       throw new IllegalStateException("FibManager registration Listener failed.", e);
 211     }
 212   }
 213
 214
 215   private InstanceIdentifier<VrfEntry> getWildCardPath() {
 216     return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 217   }
 218
 219   public DataBroker getResourceBroker() {
 220       return broker;
 221   }
 222
 223   @Override
 224   protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 225       Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 226       String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 227       LOG.info("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
 228                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 229       if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 230           createFibEntries(identifier, vrfEntry);
 231       } else {
 232           ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 233           actResource.setAction(ActionableResource.CREATE);
 234           actResource.setInstanceIdentifier(identifier);
 235           actResource.setInstance(vrfEntry);
 236           vrfEntryBufferQ.add(actResource);
 237           leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 238       }
 239       LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
 240                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 241   }
 242
 243   @Override
 244   protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 245     Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 246     String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 247     LOG.info("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
 248              rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 249     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 250         deleteFibEntries(identifier, vrfEntry);
 251     } else {
 252         ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
 253         actResource.setAction(ActionableResource.DELETE);
 254         actResource.setInstanceIdentifier(identifier);
 255         actResource.setInstance(vrfEntry);
 256         vrfEntryBufferQ.add(actResource);
 257         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 258     }
 259     LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
 260              rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
 261     leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 262   }
 263
 264   @Override
 265   protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 266     Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 267     String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 268     LOG.info("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
 269              rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 270     if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
 271         createFibEntries(identifier, update);
 272     } else {
 273         ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
 274         actResource.setAction(ActionableResource.UPDATE);
 275         actResource.setInstanceIdentifier(identifier);
 276         actResource.setInstance(update);
 277         actResource.setOldInstance(original);
 278         vrfEntryBufferQ.add(actResource);
 279     }
 280     LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
 281              rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
 282   }
 283
 284   public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 285       if (vrfEntry instanceof VrfEntry) {
 286           createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
 287       }
 288   }
 289
 290   public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) {
 291       if (vrfEntry instanceof VrfEntry) {
 292           deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 293       }
 294   }
 295
 296   public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
 297                      Object update) {
 298       if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 299           createFibEntries(tx, identifier, (VrfEntry)update);
 300       }
 301   }
 302
 303   public int getBatchSize() {
 304       return batchSize;
 305   }
 306
 307   public int getBatchInterval() {
 308       return batchInterval;
 309   }
 310
 311   public LogicalDatastoreType getDatastoreType() {
 312       return LogicalDatastoreType.CONFIGURATION;
 313   }
 314
 315   private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 316       final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 317
 318       final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 319       Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 320       Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 321
 322       final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 323       final Long vpnId = vpnInstance.getVpnId();
 324       final String rd = vrfTableKey.getRouteDistinguisher();
 325       SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 326       if (subnetRoute != null) {
 327           final long elanTag = subnetRoute.getElantag();
 328           LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 329                   rd, vrfEntry.getDestPrefix(), elanTag);
 330           if (vpnToDpnList != null) {
 331               DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 332               dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 333                       new Callable<List<ListenableFuture<Void>>>() {
 334                           @Override
 335                           public List<ListenableFuture<Void>> call() throws Exception {
 336                               WriteTransaction tx = broker.newWriteOnlyTransaction();
 337                               for (final VpnToDpnList curDpn : vpnToDpnList) {
 338                                   if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 339                                       installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
 340                                   }
 341                               }
 342                               List<ListenableFuture<Void>> futures = new ArrayList<>();
 343                               futures.add(tx.submit());
 344                               return futures;
 345                           }
 346                       });
 347           }
 348           return;
 349       }
 350
 351       if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
 352           // When it is a leaked route, the LFIB and FIB goes a bit different.
 353           installInterVpnRouteInLFib(rd, vrfEntry);
 354           return;
 355       }
 356
 357       final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 358
 359       if (vpnToDpnList != null) {
 360           DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 361           dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
 362                   new Callable<List<ListenableFuture<Void>>>() {
 363                       @Override
 364                       public List<ListenableFuture<Void>> call() throws Exception {
 365                           WriteTransaction tx = broker.newWriteOnlyTransaction();
 366                           for (VpnToDpnList vpnDpn : vpnToDpnList) {
 367                               if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
 368                                   if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 369                                       createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
 370                                   }
 371                               }
 372                           }
 373                           List<ListenableFuture<Void>> futures = new ArrayList<>();
 374                           futures.add(tx.submit());
 375                           return futures;
 376                       }
 377                   });
 378       }
 379
 380       Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(broker, rd);
 381       if ( vpnUuid.isPresent() ) {
 382           Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(broker, vpnUuid.get());
 383           if ( interVpnLink.isPresent() ) {
 384               String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
 385               if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid.get(), interVpnLink.get()) ) {
 386                   // This is an static route that points to the other endpoint of an InterVpnLink
 387                   // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 388                   installRouteInInterVpnLink(interVpnLink.get(), rd, vrfEntry, vpnId);
 389               }
 390           }
 391       }
 392   }
 393
 394
 395   /*
 396     Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 397     The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 398     provided by ResourceBatchingManager
 399    */
 400   private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 401       final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 402
 403       final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 404       Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 405       Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 406
 407       final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 408       final String rd = vrfTableKey.getRouteDistinguisher();
 409       if (vpnToDpnList != null) {
 410           for (VpnToDpnList vpnDpn : vpnToDpnList) {
 411               if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 412                   createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 413               }
 414           }
 415       }
 416   }
 417
 418     /*
 419      * Returns true if the specified nexthop is the other endpoint in an
 420      * InterVpnLink, regarding one of the VPN's point of view.
 421      */
 422     private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) {
 423         return
 424             interVpnLink != null
 425             && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
 426                      && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
 427                 || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
 428                      && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
 429     }
 430
 431
 432     // FIXME: Refactoring needed here.
 433     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 434     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 435                                    int addOrRemove) {
 436         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 437         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 438
 439         String rd = vrfTableKey.getRouteDistinguisher();
 440         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 441         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
 442             if (vpnInstance == null) {
 443                 LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
 444                 return;
 445             }
 446         } else {
 447             Preconditions.checkNotNull(vpnInstance,
 448                     "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
 449         }
 450         String vpnUuid = vpnInstance.getVpnInstanceName();
 451         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
 452                                     "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 453
 454         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
 455         // there an interVpnLink for the involved vpn in order to make learn the new route to
 456         // the other part of the inter-vpn-link.
 457
 458         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
 459         Optional<InterVpnLink> interVpnLink =
 460             (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(broker, rd)
 461                                                   : FibUtil.getInterVpnLinkByRd(broker, rd);
 462         if ( !interVpnLink.isPresent() ) {
 463             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
 464             return;
 465         }
 466
 467         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
 468         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
 469         boolean proceed = (addOrRemove == NwConstants.DEL_FLOW )
 470                           || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
 471                                && interVpnLink.get().isBgpRoutesLeaking() );
 472
 473         if ( proceed ) {
 474             String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) )
 475                                      ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
 476                                      : vpnUuid;
 477
 478             String dstVpnRd = FibUtil.getVpnRd(broker, theOtherVpnId);
 479             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 480
 481             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
 482                 InstanceIdentifier.builder(FibEntries.class)
 483                                   .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
 484                                   .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
 485                                   .build();
 486             if ( addOrRemove == NwConstants.ADD_FLOW ) {
 487                 LOG.info("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
 488                          vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
 489                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
 490                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
 491                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
 492                                                                     .setLabel(label)
 493                                                                     .setOrigin(RouteOrigin.INTERVPN.getValue())
 494                                                                     .build();
 495                 MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
 496             } else {
 497                 LOG.info("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
 498                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
 499             }
 500         }
 501     }
 502
 503     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 504         LOG.info("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 505         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 506         prefixBuilder.setDpnId(lri.getDpnId());
 507         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 508         prefixBuilder.setIpAddress(lri.getPrefix());
 509         // Increment the refCount here
 510         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 511                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
 512         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 513         if (!isPresentInList) {
 514             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 515             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 516             vpnInstanceNames.add(vpnInstanceName);
 517             builder.setVpnInstanceList(vpnInstanceNames);
 518             FibUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 519         } else {
 520             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 521         }
 522         return prefixBuilder.build();
 523     }
 524
 525     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 526                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
 527         Boolean wrTxPresent = true;
 528         if (tx == null) {
 529             wrTxPresent = false;
 530             tx = broker.newWriteOnlyTransaction();
 531         }
 532         synchronized (vrfEntry.getLabel().toString().intern()) {
 533             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 534             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 535                     vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 536
 537                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 538                     Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
 539                     if (vpnInstanceOpDataEntryOptional.isPresent()) {
 540                         String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 541                         if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 542                             updateVpnReferencesInLri(lri, vpnInstanceName, false);
 543                         }
 544                     }
 545                 }
 546                 LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 547                         vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
 548             }
 549         }
 550         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 551         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId)));
 552         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 553         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 554         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 555
 556         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 557             List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 558             // reinitialize instructions list for LFIB Table
 559             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 560
 561             actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 562             LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 563             LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
 564             LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 565
 566             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 567         }
 568         if (!wrTxPresent ) {
 569             tx.submit();
 570         }
 571     }
 572
 573     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 574         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 575         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 576         // packet is commuted from Vpn2 to Vpn1.
 577         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(broker, rd);
 578         if ( !vpnNameOpc.isPresent() ) {
 579             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 580             return;
 581         }
 582
 583         String vpnName = vpnNameOpc.get();
 584         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(broker);
 585         boolean interVpnLinkFound = false;
 586         for ( InterVpnLink interVpnLink : interVpnLinks ) {
 587             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 588             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 589                                         && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 590             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
 591                 interVpnLinkFound = true;
 592
 593                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(broker, interVpnLink.getName());
 594                 if ( !vpnLinkState.isPresent()
 595                      || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 596                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 597                              interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 598                              interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
 599                     return;
 600                 }
 601
 602                 List<BigInteger> targetDpns =
 603                     ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
 604                                          : vpnLinkState.get().getSecondEndpointState().getDpId();
 605                 int lportTag =
 606                     ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 607                                          : vpnLinkState.get().getFirstEndpointState().getLportTag();
 608
 609                 for ( BigInteger dpId : targetDpns ) {
 610                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 611
 612                     BigInteger[] metadata = new BigInteger[] {
 613                         MetaDataUtil.getMetaDataForLPortDispatcher(lportTag, FibConstants.L3VPN_SERVICE_IDENTIFIER),
 614                         MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 615                     };
 616                     List<InstructionInfo> instructions =
 617                         Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
 618                                       new InstructionInfo(InstructionType.write_metadata, metadata),
 619                                       new InstructionInfo(InstructionType.goto_table,
 620                                                           new long[] { NwConstants.L3_INTERFACE_TABLE }));
 621
 622                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
 623                                        NwConstants.ADD_FLOW, null);
 624                 }
 625
 626                 break;
 627             }
 628         }
 629
 630         if ( !interVpnLinkFound ) {
 631             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
 632                      vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
 633         }
 634     }
 635
 636
 637
 638     private void installRouteInInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 639                                             final VrfEntry vrfEntry, long vpnTag) {
 640         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 641         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 642                                     && vrfEntry.getNextHopAddressList().size() == 1);
 643
 644         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 645         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 646         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 647         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(broker, interVpnLink.getName());
 648         if ( !interVpnLinkState.isPresent() ) {
 649             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 650             return;
 651         }
 652         if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
 653             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 654                      vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList().get(0), interVpnLink.getName());
 655             return;
 656         }
 657
 658
 659         // Everything Ok
 660         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 661         List<BigInteger> targetDpns =
 662             vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 663                                : interVpnLinkState.get().getSecondEndpointState().getDpId();
 664
 665         Integer otherEndpointlportTag =
 666             vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
 667                                : interVpnLinkState.get().getFirstEndpointState().getLportTag();
 668
 669         BigInteger[] metadata = new BigInteger[] {
 670                         MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag,
 671                                                                    FibConstants.L3VPN_SERVICE_IDENTIFIER),
 672                         MetaDataUtil.getMetaDataMaskForLPortDispatcher()
 673                     };
 674         List<Instruction> instructions =
 675             Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
 676                           new InstructionInfo(InstructionType.goto_table,
 677                                               new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 678
 679         String values[] = vrfEntry.getDestPrefix().split("/");
 680         String destPrefixIpAddress = values[0];
 681         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 682
 683         List<MatchInfo> matches = new ArrayList<>();
 684         matches.add(new MatchInfo(MatchFieldType.metadata,
 685                                   new BigInteger[] { BigInteger.valueOf(vpnTag),
 686                                                      MetaDataUtil.METADATA_MASK_VRFID }));
 687         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 688
 689         if (prefixLength != 0) {
 690             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
 691                                       new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
 692         }
 693
 694         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 695         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 696         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 697         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 698                                                  COOKIE_VM_FIB_TABLE, matches, instructions);
 699
 700         for ( BigInteger dpId : targetDpns ) {
 701             mdsalManager.installFlow(dpId, flowEntity);
 702         }
 703     }
 704
 705     private void removeRouteFromInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid,
 706                                              final VrfEntry vrfEntry) {
 707
 708         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 709         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
 710                                     && vrfEntry.getNextHopAddressList().size() == 1);
 711
 712         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(broker, interVpnLink.getName());
 713         if ( !interVpnLinkState.isPresent() ) {
 714             LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName());
 715             return;
 716         }
 717
 718         // Everything Ok
 719         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
 720         List<BigInteger> targetDpns =
 721             vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
 722                                : interVpnLinkState.get().getSecondEndpointState().getDpId();
 723
 724         String nextHop = vrfEntry.getNextHopAddressList().get(0);
 725         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
 726         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
 727         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
 728                                      .setFlowName(flowRef).build();
 729
 730         for ( BigInteger dpId : targetDpns ) {
 731             mdsalManager.removeFlow(dpId, flow);
 732         }
 733
 734     }
 735
 736     private boolean isVpnFirstEndPoint(InterVpnLink interVpnLink, String vpnName) {
 737         return interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 738     }
 739
 740     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 741                                                      InstanceIdentifier<T> path) {
 742
 743         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 744
 745         Optional<T> result = Optional.absent();
 746         try {
 747             result = tx.read(datastoreType, path).get();
 748         } catch (Exception e) {
 749             throw new RuntimeException(e);
 750         }
 751
 752         return result;
 753     }
 754
 755     private void makeSubnetRouteTableMissFlow(BigInteger dpnId, int addOrRemove) {
 756         final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
 757         List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 758         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 759         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[]{}));
 760         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 761         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 762         String flowRef = getTableMissFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, NwConstants.TABLE_MISS_FLOW);
 763         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef,
 764                 NwConstants.TABLE_MISS_PRIORITY, "Subnet Route Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
 765
 766         if (addOrRemove == NwConstants.ADD_FLOW) {
 767             mdsalManager.installFlow(flowEntity);
 768         } else {
 769             mdsalManager.removeFlow(flowEntity);
 770         }
 771     }
 772
 773   private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 774     List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
 775     Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
 776     String localNextHopIP = vrfEntry.getDestPrefix();
 777
 778     if (localNextHopInfo == null) {
 779         //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 780         Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 781         if (extraRoute != null) {
 782             for (String nextHopIp : extraRoute.getNexthopIpList()) {
 783                 LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 784                 if (nextHopIp != null) {
 785                     localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
 786                     localNextHopIP = nextHopIp + "/32";
 787                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 788                     returnLocalDpnId.add(dpnId);
 789                 }
 790             }
 791         }
 792         if (localNextHopInfo == null) {
 793             /* imported routes case */
 794             synchronized (vrfEntry.getLabel().toString().intern()) {
 795                 LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
 796                 if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
 797                         vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
 798                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 799                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
 800                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 801                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 802                             if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 803                                 localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 804                             } else {
 805                                 localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 806                             }
 807                         }
 808                     }
 809                     localNextHopIP = lri.getPrefix();
 810                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 811                             vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 812                     BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
 813                     returnLocalDpnId.add(dpnId);
 814                 }
 815             }
 816         }
 817     } else {
 818         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
 819         returnLocalDpnId.add(dpnId);
 820     }
 821
 822      return returnLocalDpnId;
 823   }
 824
 825    private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
 826                                                final VrfEntry vrfEntry, Long parentVpnId){
 827        if (localNextHopInfo != null) {
 828            final BigInteger dpnId = localNextHopInfo.getDpnId();
 829            if (!isVpnPresentInDpn(rd, dpnId)) {
 830                return BigInteger.ZERO;
 831            }
 832
 833            final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
 834
 835            List<ActionInfo> actionsInfos =
 836                Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
 837            final List<InstructionInfo> instructions =
 838                Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 839            actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
 840                    new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
 841            final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 842            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 843                LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 844                        dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
 845            } else {
 846                LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
 847            }
 848            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 849            dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
 850                    new Callable<List<ListenableFuture<Void>>>() {
 851                        @Override
 852                        public List<ListenableFuture<Void>> call() throws Exception {
 853                            WriteTransaction tx = broker.newWriteOnlyTransaction();
 854                            makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 855                            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 856                                makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 857                                makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
 858                            }
 859                            List<ListenableFuture<Void>> futures = new ArrayList<>();
 860                            futures.add(tx.submit());
 861                            return futures;
 862                        }
 863                    });
 864            return dpnId;
 865        }
 866        return BigInteger.ZERO;
 867    }
 868
 869    private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
 870        InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 871        Optional<VpnToDpnList> dpnInVpn = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
 872        if (dpnInVpn.isPresent()) {
 873            return true;
 874        }
 875        return false;
 876    }
 877
 878     private LabelRouteInfo getLabelRouteInfo(Long label) {
 879         InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 880                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
 881         Optional<LabelRouteInfo> opResult = read(broker, LogicalDatastoreType.OPERATIONAL, lriIid);
 882         if (opResult.isPresent()) {
 883             return opResult.get();
 884         }
 885         return null;
 886     }
 887
 888     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
 889         LOG.info("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 890         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 891                 .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
 892         if (lri == null) {
 893             return true;
 894         }
 895         List<String> vpnInstancesList = lri.getVpnInstanceList();
 896         if (vpnInstancesList.contains(vpnInstanceName)) {
 897             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 898             vpnInstancesList.remove(vpnInstanceName);
 899         }
 900         if (vpnInstancesList.size() == 0) {
 901             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 902             FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, lriId);
 903             return true;
 904         } else {
 905             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 906             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 907             FibUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
 908         }
 909         return false;
 910     }
 911
 912     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 913                                       WriteTransaction tx) {
 914       List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
 915       actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 916
 917
 918       createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 919
 920       LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 921               dpId, label, groupId);
 922   }
 923
 924   public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 925                                                WriteTransaction tx) {
 926       List<MatchInfo> mkMatches = new ArrayList<>();
 927
 928       LOG.info("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 929
 930       // Matching metadata
 931       // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 932       mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 933
 934       List<InstructionInfo> mkInstructions = new ArrayList<>();
 935       mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 936
 937       FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 938                       getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
 939                       0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 940
 941       FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 942
 943       FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 944
 945       Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
 946       InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 947               .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 948               .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
 949       tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
 950  }
 951
 952   private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
 953     FlowEntity flowEntity;
 954     LOG.info("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
 955     List<MatchInfo> mkMatches = new ArrayList<>();
 956     // Matching metadata
 957     mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 958     flowEntity = MDSALUtil.buildFlowEntity(dpId,
 959                                            NwConstants.INTERNAL_TUNNEL_TABLE,
 960                                            getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
 961                                            5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
 962                                            COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
 963     Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
 964     FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
 965     InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
 966       .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
 967       .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 968
 969     tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
 970     LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
 971   }
 972
 973     /**
 974      * Delete local FIB entry
 975      * @param vpnId
 976      * @param rd
 977      * @param vrfEntry
 978      * @return
 979      */
 980   public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 981       List<BigInteger> returnLocalDpnId = new ArrayList<>();
 982       VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
 983       String localNextHopIP = vrfEntry.getDestPrefix();
 984
 985       if (localNextHopInfo == null) {
 986           //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 987           Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
 988           if (extra_route != null) {
 989               for (String nextHopIp : extra_route.getNexthopIpList()) {
 990                   LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 991                   if (nextHopIp != null) {
 992                       localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
 993                       localNextHopIP = nextHopIp + "/32";
 994                       BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
 995                                                                   vpnId, rd, vrfEntry, true /*isExtraRoute*/);
 996                       if (!dpnId.equals(BigInteger.ZERO)) {
 997                           returnLocalDpnId.add(dpnId);
 998                       }
 999                   }
1000               }
1001           }
1002
1003           if (localNextHopInfo == null) {
1004               /* Imported VRF entry */
1005               LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1006               if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1007                       vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1008                   VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
1009                   vpnNexthopBuilder.setDpnId(lri.getDpnId());
1010                   BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
1011                           vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1012                   if (!dpnId.equals(BigInteger.ZERO)) {
1013                       returnLocalDpnId.add(dpnId);
1014                   }
1015               }
1016           }
1017
1018
1019       } else {
1020           BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1021                                                       vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1022           if (!dpnId.equals(BigInteger.ZERO)) {
1023               returnLocalDpnId.add(dpnId);
1024           }
1025       }
1026
1027       return returnLocalDpnId;
1028   }
1029
1030     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
1031                                                 final Long vpnId, final String rd,
1032                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1033         if (localNextHopInfo != null) {
1034             final BigInteger dpnId = localNextHopInfo.getDpnId();;
1035             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1036             dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
1037                     new Callable<List<ListenableFuture<Void>>>() {
1038                         @Override
1039                         public List<ListenableFuture<Void>> call() throws Exception {
1040                             WriteTransaction tx = broker.newWriteOnlyTransaction();
1041                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1042                                     NwConstants.DEL_FLOW, tx);
1043                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1044                                 makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
1045                                         DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1046                                 removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
1047                             }
1048                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1049                             futures.add(tx.submit());
1050                             return futures;
1051                         }
1052                     });
1053             //TODO: verify below adjacency call need to be optimized (?)
1054             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
1055             return dpnId;
1056         }
1057         return BigInteger.ZERO;
1058     }
1059
1060
1061   private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
1062     return InstanceIdentifier.builder(PrefixToInterface.class)
1063         .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
1064   }
1065
1066   private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
1067     Optional<Prefixes> localNextHopInfoData =
1068         FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
1069     return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
1070   }
1071
1072     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
1073         return InstanceIdentifier.builder(VpnToExtraroute.class)
1074                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
1075                         new ExtrarouteKey(ipPrefix)).build();
1076     }
1077
1078     private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
1079         Optional<Extraroute> extraRouteInfo =
1080                 FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
1081         return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
1082
1083     }
1084
1085   private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
1086         try {
1087             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
1088                        new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
1089           RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
1090           if(!rpcResult.isSuccessful()) {
1091               LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
1092           } else {
1093               return rpcResult.getResult().getTunnelType();
1094           }
1095
1096       } catch (InterruptedException | ExecutionException e) {
1097           LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
1098       }
1099
1100   return null;
1101
1102   }
1103     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1104                                       final VrfEntry vrfEntry, WriteTransaction tx) {
1105         Boolean wrTxPresent = true;
1106         if (tx == null) {
1107             wrTxPresent = false;
1108             tx = broker.newWriteOnlyTransaction();
1109         }
1110         String rd = vrfTableKey.getRouteDistinguisher();
1111         LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
1112                     vrfEntry.getDestPrefix(), rd, tx);
1113         /********************************************/
1114         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1115
1116         if (tunnelInterfaceList.isEmpty()) {
1117             LOG.error("Could not get interface for nexthop: {} in vpn {}",
1118                     vrfEntry.getNextHopAddressList(), rd);
1119             LOG.warn("Failed to add Route: {} in vpn: {}",
1120                     vrfEntry.getDestPrefix(), rd);
1121             return;
1122         }
1123
1124         for (String tunnelInterface : tunnelInterfaceList) {
1125             List<InstructionInfo> instructions = new ArrayList<>();
1126             List<ActionInfo> actionInfos = new ArrayList<>();
1127             Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
1128             if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
1129                 LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1130                 actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
1131                 actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
1132             } else {
1133                 int label = vrfEntry.getLabel().intValue();
1134                 BigInteger tunnelId;
1135                 // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
1136                 if (tunnel_type.equals(TunnelTypeVxlan.class)) {
1137                     tunnelId = BigInteger.valueOf(label);
1138                 } else {
1139                     tunnelId = BigInteger.valueOf(label);
1140                 }
1141
1142                 LOG.debug("adding set tunnel id action for label {}", label);
1143                 actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
1144             }
1145             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
1146             if(egressActions.isEmpty()){
1147                 LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
1148                 return;
1149             }
1150             actionInfos.addAll(egressActions);
1151             instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
1152             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1153         }
1154         if(!wrTxPresent ){
1155             tx.submit();
1156         }
1157         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1158     }
1159
1160     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1161         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1162         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
1163         if (dpnInVpn.isPresent()) {
1164             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1165             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1166
1167             if (vpnInterfaces.remove(currVpnInterface)) {
1168                 if (vpnInterfaces.isEmpty()) {
1169                   LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1170                   FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id);
1171                   cleanUpDpnForVpn(dpnId, vpnId, rd);
1172                 } else {
1173                   LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1174                     FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
1175                             VpnInterfaces.class,
1176                             new VpnInterfacesKey(intfName)));
1177                 }
1178             }
1179         }
1180     }
1181
1182   private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1183     /* Get interface info from prefix to interface mapping;
1184         Use the interface info to get the corresponding vpn interface op DS entry,
1185         remove the adjacency corresponding to this fib entry.
1186         If adjacency removed is the last adjacency, clean up the following:
1187          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1188          - prefix to interface entry
1189          - vpn interface op DS
1190      */
1191       LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1192       Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
1193       Extraroute extraRoute = null;
1194       if (prefixInfo == null) {
1195           extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1196           if(extraRoute != null) {
1197               for (String nextHopIp : extraRoute.getNexthopIpList()) {
1198                   LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1199
1200                   if (nextHopIp != null) {
1201                       prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
1202                       checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1203                   }
1204               }
1205           }
1206           if (prefixInfo == null) {
1207               LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1208               if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1209                       vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1210                   PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1211                   prefixBuilder.setDpnId(lri.getDpnId());
1212                   prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1213                   prefixBuilder.setIpAddress(lri.getPrefix());
1214                   prefixInfo = prefixBuilder.build();
1215                   LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1216                           vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1217                   checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1218               }
1219           }
1220       } else {
1221           checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1222       }
1223   }
1224
1225   private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1226                                         final VrfEntry vrfEntry, final Extraroute extraRoute) {
1227
1228       if (prefixInfo == null) {
1229           LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1230           return; //Don't have any info for this prefix (shouldn't happen); need to return
1231       }
1232
1233       String ifName = prefixInfo.getVpnInterfaceName();
1234       DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1235       dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1236               new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1237   }
1238
1239   private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1240       Prefixes prefixInfo;
1241       Long vpnId;
1242       String rd;
1243       VrfEntry vrfEntry;
1244       Extraroute extraRoute;
1245
1246       public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1247                                        final VrfEntry vrfEntry, final Extraroute extraRoute) {
1248           this.prefixInfo = prefixInfo;
1249           this.vpnId = vpnId;
1250           this.rd= rd;
1251           this.vrfEntry= vrfEntry;
1252           this.extraRoute = extraRoute;
1253       }
1254
1255       @Override
1256       public List<ListenableFuture<Void>> call() throws Exception {
1257           // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1258           // to call the respective helpers.
1259           String ifName = prefixInfo.getVpnInterfaceName();
1260           WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
1261           Optional<VpnInterface> optvpnInterface = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1262                   FibUtil.getVpnInterfaceIdentifier(ifName));
1263           if (optvpnInterface.isPresent()) {
1264               long associatedVpnId = FibUtil.getVpnId(broker, optvpnInterface.get().getVpnInstanceName());
1265               if (vpnId != associatedVpnId) {
1266                   LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1267                           vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1268                   LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
1269                   FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1270                           FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1271                   LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1272                   return null;
1273               } else {
1274                   LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1275                           vrfEntry.getDestPrefix(), associatedVpnId);
1276               }
1277           }
1278           if (extraRoute != null) {
1279               FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
1280                       FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
1281           }
1282           Optional<Adjacencies> optAdjacencies = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
1283                   FibUtil.getAdjListPath(ifName));
1284           int numAdj = 0;
1285           if (optAdjacencies.isPresent()) {
1286               numAdj = optAdjacencies.get().getAdjacency().size();
1287           }
1288           //remove adjacency corr to prefix
1289           if (numAdj > 1) {
1290               LOG.trace("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
1291               FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
1292                       FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1293           }
1294           if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
1295               //clean up the vpn interface from DpnToVpn list
1296               LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1297               FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
1298                       FibUtil.getVpnInterfaceIdentifier(ifName));
1299           }
1300
1301           synchronized (vrfEntry.getLabel().toString().intern()) {
1302               LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1303               if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
1304                       vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1305                   Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
1306                   String vpnInstanceName = "";
1307                   if (vpnInstanceOpDataEntryOptional.isPresent()) {
1308                       vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1309                   }
1310                   boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
1311                   if (lriRemoved) {
1312                       String parentRd = lri.getParentVpnRd();
1313                       FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1314                               FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1315                   }
1316               }
1317           }
1318           CheckedFuture<Void, TransactionCommitFailedException> futures = writeTxn.submit();
1319           try {
1320               futures.get();
1321           } catch (InterruptedException | ExecutionException e) {
1322               LOG.error("Error cleaning up interface {} on vpn {}", ifName, vpnId);
1323               throw new RuntimeException(e.getMessage());
1324           }
1325           return null;
1326       }
1327   }
1328
1329   private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1330     final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1331
1332     final String rd  = vrfTableKey.getRouteDistinguisher();
1333     final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1334     if (vpnInstance == null) {
1335         LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1336         return;
1337     }
1338     final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1339     long elanTag = 0L;
1340     SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1341     if (subnetRoute != null) {
1342         elanTag = subnetRoute.getElantag();
1343         LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1344                 rd, vrfEntry.getDestPrefix(), elanTag);
1345         if (vpnToDpnList != null) {
1346             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1347             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1348                     new Callable<List<ListenableFuture<Void>>>() {
1349                         @Override
1350                         public List<ListenableFuture<Void>> call() throws Exception {
1351                             WriteTransaction tx = broker.newWriteOnlyTransaction();
1352
1353                             for (final VpnToDpnList curDpn : vpnToDpnList) {
1354
1355                                 makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1356                                         vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1357                                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1358                                     makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
1359                                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1360                                 }
1361                             }
1362                             List<ListenableFuture<Void>> futures = new ArrayList<>();
1363                             futures.add(tx.submit());
1364                             return futures;
1365                         }
1366                     });
1367         }
1368         synchronized (vrfEntry.getLabel().toString().intern()) {
1369             LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1370             if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1371                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
1372                 String vpnInstanceName = "";
1373                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
1374                     vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1375                 }
1376                 boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
1377                 if (lriRemoved) {
1378                     String parentRd = lri.getParentVpnRd();
1379                     FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1380                             FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1381                     LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1382                             vrfEntry.getDestPrefix());
1383                 }
1384             }
1385         }
1386         return;
1387     }
1388
1389     final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1390                                                           vrfTableKey.getRouteDistinguisher(), vrfEntry);
1391     if (vpnToDpnList != null) {
1392         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1393         dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
1394                 new Callable<List<ListenableFuture<Void>>>() {
1395                     @Override
1396                     public List<ListenableFuture<Void>> call() throws Exception {
1397                         WriteTransaction tx = broker.newWriteOnlyTransaction();
1398
1399                         if (localDpnIdList.size() <= 0) {
1400                             for (VpnToDpnList curDpn : vpnToDpnList) {
1401                                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1402                                     if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1403                                         deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1404                                     }
1405                                 } else {
1406                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1407                                 }
1408                             }
1409                         } else {
1410                             for (BigInteger localDpnId : localDpnIdList) {
1411                                 for (VpnToDpnList curDpn : vpnToDpnList) {
1412                                     if (!curDpn.getDpnId().equals(localDpnId)) {
1413                                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1414                                             if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1415                                                 deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1416                                             }
1417                                         } else {
1418                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
1419                                         }
1420                                     }
1421                                 }
1422                             }
1423                         }
1424                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1425                         futures.add(tx.submit());
1426                         return futures;
1427                     }
1428                 });
1429     }
1430
1431     //The flow/group entry has been deleted from config DS; need to clean up associated operational
1432     //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1433     cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1434
1435     // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1436     // of the interVpnLink.
1437     Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(broker, rd);
1438     if ( vpnUuid.isPresent() ) {
1439         Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(broker, vpnUuid.get());
1440         String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
1441
1442         if ( interVpnLink.isPresent()
1443              && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
1444                    && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
1445                   || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
1446                       && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
1447             // This is route that points to the other endpoint of an InterVpnLink
1448             // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
1449             removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
1450         }
1451     }
1452
1453   }
1454
1455   /*
1456     Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1457     The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1458     provided by ResourceBatchingManager
1459    */
1460   private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1461       final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1462
1463       final String rd  = vrfTableKey.getRouteDistinguisher();
1464       final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1465       if (vpnInstance == null) {
1466           LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1467           return;
1468       }
1469       final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1470       if (vpnToDpnList != null) {
1471           for (VpnToDpnList curDpn : vpnToDpnList) {
1472               if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1473                   deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
1474               }
1475           }
1476       }
1477   }
1478
1479   public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1480                                 final long vpnId, final VrfTablesKey vrfTableKey,
1481                                 final VrfEntry vrfEntry, WriteTransaction tx) {
1482
1483       Boolean wrTxPresent = true;
1484       if (tx == null) {
1485           wrTxPresent = false;
1486           tx = broker.newWriteOnlyTransaction();
1487       }
1488
1489     LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1490     String rd = vrfTableKey.getRouteDistinguisher();
1491
1492       if(localDpnId != null) {
1493           // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1494           deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1495           return;
1496       }
1497
1498       // below two reads are kept as is, until best way is found to identify dpnID
1499       VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1500       Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1501
1502       if (localNextHopInfo == null && extraRoute != null) {
1503           // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1504           for (String nextHopIp : extraRoute.getNexthopIpList()) {
1505               localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
1506               checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1507           }
1508       } else {
1509               checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1510       }
1511       if(!wrTxPresent ){
1512           tx.submit();
1513       }
1514   }
1515
1516     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1517                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
1518         boolean isRemoteRoute = true;
1519         if (localNextHopInfo != null) {
1520             isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId()));
1521         }
1522         if (isRemoteRoute) {
1523             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1524             return true;
1525         } else {
1526             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
1527             return false;
1528         }
1529     }
1530
1531     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
1532             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1533             LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1534     }
1535
1536   private long get
1537           (byte[] rawIpAddress) {
1538     return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1539             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1540   }
1541
1542   private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1543                                   List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1544     Boolean wrTxPresent = true;
1545     if (tx == null) {
1546       wrTxPresent = false;
1547       tx = broker.newWriteOnlyTransaction();
1548     }
1549
1550     LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1551     String values[] = vrfEntry.getDestPrefix().split("/");
1552     String ipAddress = values[0];
1553     int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1554     if (addOrRemove == NwConstants.ADD_FLOW) {
1555         LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1556     } else {
1557         LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1558     }
1559     InetAddress destPrefix;
1560     try {
1561       destPrefix = InetAddress.getByName(ipAddress);
1562     } catch (UnknownHostException e) {
1563       LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1564       return;
1565     }
1566
1567     List<MatchInfo> matches = new ArrayList<>();
1568
1569     matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
1570         BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
1571
1572     matches.add(new MatchInfo(MatchFieldType.eth_type,
1573                               new long[] { NwConstants.ETHTYPE_IPV4 }));
1574
1575     if(prefixLength != 0) {
1576       matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
1577           destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
1578     }
1579     int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1580     String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1581     FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
1582                                                       COOKIE_VM_FIB_TABLE, matches, instructions);
1583
1584     Flow flow = flowEntity.getFlowBuilder().build();
1585     String flowId = flowEntity.getFlowId();
1586     FlowKey flowKey = new FlowKey( new FlowId(flowId));
1587     Node nodeDpn = buildDpnNode(dpId);
1588
1589     InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1590             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1591             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1592     if (addOrRemove == NwConstants.ADD_FLOW) {
1593         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1594     } else {
1595         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1596     }
1597
1598     if(!wrTxPresent ){
1599       tx.submit();
1600     }
1601   }
1602
1603     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1604     private Node buildDpnNode(BigInteger dpnId) {
1605         NodeId nodeId = new NodeId("openflow:" + dpnId);
1606         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1607
1608         return nodeDpn;
1609     }
1610
1611     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1612                                     int addOrRemove, WriteTransaction tx) {
1613         Boolean wrTxPresent = true;
1614         if (tx == null) {
1615             wrTxPresent = false;
1616             tx = broker.newWriteOnlyTransaction();
1617         }
1618
1619         List<MatchInfo> matches = new ArrayList<MatchInfo>();
1620         matches.add(new MatchInfo(MatchFieldType.eth_type,
1621                                   new long[] { NwConstants.ETHTYPE_MPLS_UC }));
1622         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
1623
1624         // Install the flow entry in L3_LFIB_TABLE
1625         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1626
1627         FlowEntity flowEntity;
1628         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1629                                                COOKIE_VM_LFIB_TABLE, matches, instructions);
1630         Flow flow = flowEntity.getFlowBuilder().build();
1631         String flowId = flowEntity.getFlowId();
1632         FlowKey flowKey = new FlowKey( new FlowId(flowId));
1633         Node nodeDpn = buildDpnNode(dpId);
1634         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1635                 .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1636                 .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1637
1638         if (addOrRemove == NwConstants.ADD_FLOW) {
1639             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
1640         } else {
1641             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1642         }
1643         if(!wrTxPresent ){
1644             tx.submit();
1645         }
1646         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",dpId, label, instructions );
1647     }
1648
1649   private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
1650     LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
1651     try {
1652       nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
1653     } catch (NullPointerException e) {
1654       LOG.trace("", e);
1655     }
1656   }
1657
1658   public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd) {
1659       LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1660       InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1661       synchronized (rd.intern()) {
1662           final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1663           if (vrfTable.isPresent()) {
1664               DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1665               dataStoreCoordinator.enqueueJob("FIB" + vpnId + dpnId.toString(),
1666                       new Callable<List<ListenableFuture<Void>>>() {
1667                           @Override
1668                           public List<ListenableFuture<Void>> call() throws Exception {
1669                               WriteTransaction tx = broker.newWriteOnlyTransaction();
1670                               for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1671
1672                                   SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1673                                   if (subnetRoute != null) {
1674                                       long elanTag = subnetRoute.getElantag();
1675                                       installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1676                                       continue;
1677                                   }
1678                                   if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
1679                                       LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
1680                                       if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
1681                                           if (lri.getDpnId().equals(dpnId)) {
1682                                               createLocalFibEntry(vpnId, rd, vrfEntry);
1683                                               continue;
1684                                           }
1685                                       }
1686                                   }
1687                                   // Passing null as we don't know the dpn
1688                                   // to which prefix is attached at this point
1689                                   createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1690                               }
1691                               //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1692                               List<ListenableFuture<Void>> futures = new ArrayList<>();
1693                               futures.add(tx.submit());
1694                               return futures;
1695                           }
1696                       });
1697           }
1698       }
1699   }
1700
1701   public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd, final String localNextHopIp, final String remoteNextHopIp) {
1702     LOG.trace(  "dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
1703                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1704     InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1705     synchronized (rd.intern()) {
1706       final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1707       if (vrfTable.isPresent()) {
1708           DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1709           dataStoreCoordinator.enqueueJob(" FIB + on Dpn , rd "
1710                             + rd.toString() + "localNextHopIp "
1711                             + localNextHopIp + "remoteNextHopIP"
1712                             + remoteNextHopIp + "vpnId "
1713                             + vpnId + "dpnId" + dpnId,
1714                   new Callable<List<ListenableFuture<Void>>>() {
1715                       @Override
1716                       public List<ListenableFuture<Void>> call() throws Exception {
1717                           WriteTransaction writeTransaction = broker.newWriteOnlyTransaction();
1718                           List<ListenableFuture<Void>> futures = new ArrayList<>();
1719                           LOG.trace("populate FIB starts on Dpn " + dpnId
1720                                   + "rd  " + rd.toString()
1721                                   + "localNextHopIp " + localNextHopIp
1722                                   + "remoteNextHopIp" + remoteNextHopIp
1723                                   + "vpnId " + vpnId );
1724
1725                           for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1726                                 LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
1727
1728                               if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
1729                                   if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1730                                       LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
1731                                       createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1732                                   }
1733                               }
1734
1735                               if ((vrfEntry.getOrigin().equals(RouteOrigin.CONNECTED.getValue())) ||
1736                                   (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue()))) {
1737                                   String destPfx = vrfEntry.getDestPrefix();
1738                                   BigInteger dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
1739                                   if (dpnIdForPrefix == null) {
1740                                       LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
1741                                                 vrfEntry.getDestPrefix());
1742                                       continue;
1743                                   }
1744                                   int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
1745                                   if (sameDpnId != 0) {
1746                                       LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
1747                                               dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
1748                                      continue;
1749                                   }
1750
1751                                   // Passing null as we don't know the dpn
1752                                   // to which prefix is attached at this point
1753                                   InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
1754
1755
1756                                   vrfEntry.getNextHopAddressList().add(localNextHopIp);
1757                                   VrfEntry newVrfEntry =
1758                                           new VrfEntryBuilder(vrfEntry).setNextHopAddressList(vrfEntry.getNextHopAddressList()).build();
1759                                   // Just update the VrfEntry
1760                                   FibUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION,
1761                                           vrfEntryId, newVrfEntry);
1762                                   vrfEntry = getVrfEntry(broker, rd, destPfx);
1763                                   LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
1764                               }
1765                           }
1766                           futures.add(writeTransaction.submit());
1767                           LOG.trace("populate FIB ends on Dpn " + dpnId
1768                                   + "rd  " + rd.toString()
1769                                   + "localNextHopIp " + localNextHopIp
1770                                   + "remoteNextHopIp" + remoteNextHopIp
1771                                   + "vpnId " + vpnId );
1772                           return futures;
1773                       }
1774                   });
1775       }
1776     }
1777   }
1778
1779   public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId, final long vpnId, final String  rd, final String destPrefix , final String localNextHopIP,
1780                                 final String remoteNextHopIp) {
1781
1782       DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1783       dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
1784                                         + "local dpid" + localDpnId
1785                                         + "remote dpid" + remoteDpnId
1786                                         + "vpnId" + vpnId
1787                                         + "localNHIp" + localNextHopIP
1788                                         + "remoteNHIp" + remoteNextHopIp,
1789               new Callable<List<ListenableFuture<Void>>>() {
1790                   @Override
1791                   public List<ListenableFuture<Void>> call() throws Exception {
1792                       List<ListenableFuture<Void>> futures = new ArrayList<>();
1793                       WriteTransaction writeTransaction = broker.newWriteOnlyTransaction();
1794                       VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
1795                       VrfEntry vrfEntry = getVrfEntry(broker, rd, destPrefix);
1796                       if (vrfEntry == null)
1797                           return futures;
1798                       LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
1799                                 "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
1800                                 action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
1801                       if (action == true) {
1802                           vrfEntry = getVrfEntry(broker, rd, destPrefix);
1803                           LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
1804                           createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1805                       } else {
1806                           vrfEntry = getVrfEntry(broker, rd, destPrefix);
1807                           LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
1808                           deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
1809                       }
1810                       futures.add(writeTransaction.submit());
1811                       return futures;
1812                   }
1813               });
1814   }
1815
1816   public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd) {
1817       LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
1818       InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1819       synchronized (rd.intern()) {
1820           final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1821           if (vrfTable.isPresent()) {
1822               DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1823               dataStoreCoordinator.enqueueJob("FIB" + vpnId + dpnId.toString(),
1824                       new Callable<List<ListenableFuture<Void>>>() {
1825                           WriteTransaction tx = broker.newWriteOnlyTransaction();
1826                           @Override
1827                           public List<ListenableFuture<Void>> call() throws Exception {
1828                               for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1829                             /* Handle subnet routes here */
1830                                   SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1831                                   if (subnetRoute != null) {
1832                                       LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
1833                                               dpnId, rd);
1834                                       makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1835                                       makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1836                                       LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
1837                                               vrfEntry.getDestPrefix());
1838                                       continue;
1839                                   }
1840                                   // Passing null as we don't know the dpn
1841                                   // to which prefix is attached at this point
1842                                   deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1843                               }
1844                               List<ListenableFuture<Void>> futures = new ArrayList<>();
1845                               futures.add(tx.submit());
1846                               return futures;
1847                           }
1848
1849                       });
1850           }
1851
1852       }
1853   }
1854
1855   public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
1856                                final String localNextHopIp, final String remoteNextHopIp) {
1857     LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
1858                 " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
1859                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1860     InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1861     synchronized (rd.intern()) {
1862       final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
1863       if (vrfTable.isPresent()) {
1864           DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1865           dataStoreCoordinator.enqueueJob(" FIB + on Dpn " + dpnId
1866                                             + rd + rd.toString()
1867                                             + "localNextHopIp " + localNextHopIp
1868                                             + "remoteNextHopIP" + remoteNextHopIp
1869                                             + "vpnId " + vpnId
1870                                             + "dpnId" + dpnId,
1871                   new Callable<List<ListenableFuture<Void>>>() {
1872                       @Override
1873                       public List<ListenableFuture<Void>> call() throws Exception {
1874                           WriteTransaction writeTransaction = broker.newWriteOnlyTransaction();
1875                           List<ListenableFuture<Void>> futures = new ArrayList<>();
1876                           LOG.trace("cleanup FIB starts on Dpn " + dpnId
1877                                     + "rd  " + rd.toString()
1878                                     + "localNextHopIp " + localNextHopIp
1879                                     + "remoteNextHopIp" + remoteNextHopIp
1880                                     + "vpnId " + vpnId );
1881
1882                           for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1883                               LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
1884                               if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1885                                   LOG.trace(" deleting remote FIB entry {}", vrfEntry);
1886                                   deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
1887                               }
1888
1889                               if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
1890                                   LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
1891                                             vrfEntry.getDestPrefix(), dpnId);
1892                                   String destPfx = vrfEntry.getDestPrefix();
1893                                   InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
1894                                   List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
1895                                   newList.remove(localNextHopIp);
1896                                  VrfEntry newVrfEntry =
1897                                           new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
1898                                   FibUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION,
1899                                           vrfEntryId, newVrfEntry);
1900                                   vrfEntry = getVrfEntry(broker, rd, destPfx);
1901                                   LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
1902                               }
1903                           }
1904                           futures.add(writeTransaction.submit());
1905                           LOG.trace("cleanup FIB ends on Dpn " + dpnId
1906                                     + "rd  " + rd.toString()
1907                                     + "localNextHopIp " + localNextHopIp
1908                                     + "remoteNextHopIp" + remoteNextHopIp
1909                                     + "vpnId " + vpnId );
1910                           return futures;
1911                       }
1912                   });
1913
1914       }
1915     }
1916   }
1917
1918   public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
1919     InstanceIdentifierBuilder<VrfTables> idBuilder =
1920         InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1921     InstanceIdentifier<VrfTables> id = idBuilder.build();
1922     return id;
1923   }
1924
1925   private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
1926     return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1927         .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
1928         .append(priority).toString();
1929   }
1930
1931   private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
1932     return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
1933         .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
1934         .append(rd).append(NwConstants.FLOWID_SEPARATOR)
1935         .append(priority).append(NwConstants.FLOWID_SEPARATOR)
1936         .append(destPrefix.getHostAddress()).toString();
1937   }
1938
1939   private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
1940       return new StringBuilder(64).append(FLOWID_PREFIX)
1941                                   .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
1942                                   .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
1943                                   .append(nextHop).toString();
1944   }
1945
1946   protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId, final VrfEntry vrfEntry,
1947                                           String rd) {
1948     List<String> adjacencyList = new ArrayList<>();
1949     List<String> prefixIpList = new ArrayList<>();
1950     LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}", remoteDpnId, vpnId, vrfEntry);
1951     try {
1952         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
1953             Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1954             if (extra_route == null) {
1955                 prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1956             } else {
1957                 prefixIpList = new ArrayList<>();
1958                 for (String extraRouteIp : extra_route.getNexthopIpList()) {
1959                     prefixIpList.add(extraRouteIp + "/32");
1960                 }
1961             }
1962         } else {
1963             prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
1964         }
1965
1966         for (String prefixIp : prefixIpList) {
1967             for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
1968                 LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
1969                 String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId, prefixIp, nextHopIp);
1970                 if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
1971                     adjacencyList.add(adjacency);
1972                 }
1973             }
1974         }
1975     } catch (NullPointerException e) {
1976       LOG.trace("", e);
1977     }
1978     return adjacencyList;
1979   }
1980
1981   protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
1982     InstanceIdentifier<VpnInstanceOpDataEntry> id = InstanceIdentifier.create(VpnInstanceOpData.class).child(
1983         VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
1984     Optional<VpnInstanceOpDataEntry> vpnInstanceOpData = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
1985     return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
1986   }
1987
1988     public void processNodeAdd(BigInteger dpnId) {
1989         LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId);
1990         makeTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1991         makeL3IntfTblMissFlow(dpnId, NwConstants.ADD_FLOW);
1992         makeSubnetRouteTableMissFlow(dpnId, NwConstants.ADD_FLOW);
1993     }
1994
1995     private void makeTableMissFlow(BigInteger dpnId, int addOrRemove) {
1996         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
1997         // Instruction to goto L3 InterfaceTable
1998         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
1999         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE }));
2000         List<MatchInfo> matches = new ArrayList<MatchInfo>();
2001         FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_LFIB_TABLE,
2002                 getTableMissFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW),
2003                 NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
2004
2005         FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE,
2006                                                              getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE,
2007                                                                                  NwConstants.TABLE_MISS_FLOW),
2008                                                              NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow",
2009                                                              0, 0, COOKIE_VM_FIB_TABLE,
2010                                                              matches, instructions);
2011
2012         if (addOrRemove == NwConstants.ADD_FLOW) {
2013             LOG.debug("Invoking MDSAL to install Table Miss Entries");
2014             mdsalManager.installFlow(flowEntityLfib);
2015             mdsalManager.installFlow(flowEntityFib);
2016         } else {
2017             mdsalManager.removeFlow(flowEntityLfib);
2018             mdsalManager.removeFlow(flowEntityFib);
2019
2020         }
2021     }
2022
2023     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2024         return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
2025                 .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
2026                 .append(FLOWID_PREFIX).toString();
2027     }
2028
2029   /*
2030    * Install flow entry in protocol table to forward mpls
2031    * coming through gre tunnel to LFIB table.
2032    */
2033   private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2034     final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
2035     // Instruction to goto L3 InterfaceTable
2036     List<InstructionInfo> instructions = new ArrayList<>();
2037     instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
2038     List<MatchInfo> matches = new ArrayList<MatchInfo>();
2039     matches.add(new MatchInfo(MatchFieldType.eth_type,
2040                               new long[] { NwConstants.ETHTYPE_MPLS_UC }));
2041     FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_PROTOCOL_TABLE,
2042                                                           getTableMissFlowRef(dpnId, NwConstants.L3_PROTOCOL_TABLE,
2043                                                                   NwConstants.L3_LFIB_TABLE),
2044                                                           DEFAULT_FIB_FLOW_PRIORITY,
2045                                                           "Protocol Table For LFIB",
2046                                                           0, 0,
2047                                                           COOKIE_PROTOCOL_TABLE,
2048                                                           matches, instructions);
2049
2050     if (addOrRemove == NwConstants.ADD_FLOW) {
2051       LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2052       mdsalManager.installFlow(flowEntityToLfib);
2053     } else {
2054       mdsalManager.removeFlow(flowEntityToLfib);
2055     }
2056   }
2057
2058   public List<String> printFibEntries() {
2059     List<String> result = new ArrayList<String>();
2060     result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2061     result.add("-------------------------------------------------------------------");
2062     InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2063     Optional<FibEntries> fibEntries = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
2064     if (fibEntries.isPresent()) {
2065         List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2066         for (VrfTables vrfTable : vrfTables) {
2067             for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2068                 for (String nextHop : vrfEntry.getNextHopAddressList()) {
2069                     result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", vrfTable.getRouteDistinguisher(),
2070                             vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
2071                 }
2072                 if (vrfEntry.getNextHopAddressList().isEmpty()) {
2073                     result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", vrfTable.getRouteDistinguisher(),
2074                             vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
2075                 }
2076             }
2077         }
2078     }
2079     return result;
2080   }
2081
2082   private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) {
2083     List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
2084     List<MatchInfo> matches = new ArrayList<MatchInfo>();
2085     final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
2086     // Instruction to goto L3 InterfaceTable
2087
2088     List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
2089     actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
2090         Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
2091     instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
2092     //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
2093
2094     FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
2095             getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
2096             NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
2097     if (addOrRemove == NwConstants.ADD_FLOW) {
2098       LOG.info("Invoking MDSAL to install L3 interface Table Miss Entries");
2099       mdsalManager.installFlow(flowEntityL3Intf);
2100     } else {
2101       mdsalManager.removeFlow(flowEntityL3Intf);
2102     }
2103   }
2104
2105     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2106         InstanceIdentifier<VrfEntry> vrfEntryId =
2107                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2108                             child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2109         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2110         if (vrfEntry.isPresent())  {
2111             return (vrfEntry.get());
2112         }
2113         return null;
2114     }
2115
2116     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2117         InstanceIdentifier<VrfEntry> vrfEntryId =
2118                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
2119                         child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2120         return vrfEntryId;
2121     }
2122 }