vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17
  18 import java.math.BigInteger;
  19 import java.net.Inet4Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.BlockingQueue;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.function.Consumer;
  31
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  37 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  38 import org.opendaylight.genius.mdsalutil.ActionInfo;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.MatchInfo;
  43 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  62 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.ServiceIndex;
  72 import org.opendaylight.genius.utils.batching.ActionableResource;
  73 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  74 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  75 import org.opendaylight.genius.utils.batching.ResourceHandler;
  76 import org.opendaylight.genius.utils.batching.SubTransaction;
  77 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  78 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  79 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  82 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  85 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 138 import org.opendaylight.yangtools.yang.binding.DataObject;
 139 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 140 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 141 import org.slf4j.Logger;
 142 import org.slf4j.LoggerFactory;
 143
 144 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 145     implements AutoCloseable, ResourceHandler {
 146
 147     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 148     private static final String FLOWID_PREFIX = "L3.";
 149     private static final int BATCH_SIZE = 1000;
 150     private static final int PERIODICITY = 500;
 151     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 152     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 153     private static final int LFIB_INTERVPN_PRIORITY = 15;
 154     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 155
 156     private final DataBroker dataBroker;
 157     private final IMdsalApiManager mdsalManager;
 158     private final NexthopManager nextHopManager;
 159     private final OdlInterfaceRpcService interfaceManager;
 160     private final IdManagerService idManager;
 161     private final IVpnLinkService ivpnLinkService;
 162     List<SubTransaction> transactionObjects;
 163
 164     private static Integer batchSize;
 165     private static Integer batchInterval;
 166
 167     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 168     private final ResourceBatchingManager resourceBatchingManager;
 169
 170     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 171                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 172                             final IdManagerService idManager, final IVpnLinkService ivpnLinkService) {
 173         super(VrfEntry.class, VrfEntryListener.class);
 174         this.dataBroker = dataBroker;
 175         this.mdsalManager = mdsalApiManager;
 176         this.nextHopManager = nexthopManager;
 177         this.interfaceManager = interfaceManager;
 178         this.idManager = idManager;
 179         this.ivpnLinkService = ivpnLinkService;
 180
 181         batchSize = Integer.getInteger("batch.size", BATCH_SIZE);
 182         batchInterval = Integer.getInteger("batch.wait.time", PERIODICITY);
 183         resourceBatchingManager = ResourceBatchingManager.getInstance();
 184         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 185         transactionObjects = new ArrayList<>();
 186     }
 187
 188     public void start() {
 189         LOG.info("{} start", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     protected VrfEntryListener getDataTreeChangeListener() {
 195         return VrfEntryListener.this;
 196     }
 197
 198     @Override
 199     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 200         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 201     }
 202
 203     @Override
 204     public DataBroker getResourceBroker() {
 205         return dataBroker;
 206     }
 207
 208     public NexthopManager getNextHopManager() {
 209         return this.nextHopManager;
 210     }
 211
 212     @Override
 213     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 214         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 215         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 216         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 217                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 218         if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Vxlan)) {
 219             LOG.info("EVPN flows need to be programmed.");
 220             EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier,
 221                     vrfEntry, dataBroker, this);
 222             evpnVrfEntryProcessor.installFlows();
 223         } else {
 224             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 225                 createFibEntries(identifier, vrfEntry);
 226             } else {
 227                 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 228                 actResource.setAction(ActionableResource.CREATE);
 229                 actResource.setInstanceIdentifier(identifier);
 230                 actResource.setInstance(vrfEntry);
 231                 vrfEntryBufferQ.add(actResource);
 232             }
 233         }
 234         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 235         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 236             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 237     }
 238
 239     @Override
 240     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 241         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 242         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 243         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 244                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 245         if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Vxlan)) {
 246             LOG.info("EVPN flows to be deleted");
 247             EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier, vrfEntry,
 248                     dataBroker, this);
 249             evpnVrfEntryProcessor.removeFlows();
 250         } else {
 251             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 252                 deleteFibEntries(identifier, vrfEntry);
 253             } else {
 254                 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 255                 actResource.setAction(ActionableResource.DELETE);
 256                 actResource.setInstanceIdentifier(identifier);
 257                 actResource.setInstance(vrfEntry);
 258                 vrfEntryBufferQ.add(actResource);
 259             }
 260         }
 261         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 262         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 263             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 264     }
 265
 266     @Override
 267     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 268         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 269
 270         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 271         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 272         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {}",
 273             rd, update.getDestPrefix(), update.getRoutePaths());
 274         // Handle BGP Routes first
 275         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 276             ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
 277             actResource.setAction(ActionableResource.UPDATE);
 278             actResource.setInstanceIdentifier(identifier);
 279             actResource.setInstance(update);
 280             actResource.setOldInstance(original);
 281             vrfEntryBufferQ.add(actResource);
 282             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 283                 rd, update.getDestPrefix(), update.getRoutePaths());
 284             return;
 285         }
 286
 287         // Handle Vpn Interface driven Routes next (ie., STATIC and LOCAL)
 288         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(update.getOrigin()))) {
 289             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 290             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 291             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 292
 293             // If original VRF Entry had nexthop null , but update VRF Entry
 294             // has nexthop , route needs to be created on remote Dpns
 295             if (((originalRoutePath == null) || (originalRoutePath.isEmpty())
 296                 && (updateRoutePath != null) && (!updateRoutePath.isEmpty()))) {
 297                 // TODO(vivek): Though ugly, Not handling this code now, as each
 298                 // tep add event will invoke flow addition
 299                 LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 300                     update.getDestPrefix());
 301                 return;
 302             }
 303
 304             // If original VRF Entry had valid nexthop , but update VRF Entry
 305             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 306             if (((updateRoutePath == null) || (updateRoutePath.isEmpty())
 307                 && (originalRoutePath != null) && (!originalRoutePath.isEmpty()))) {
 308                 LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 309                     update.getDestPrefix());
 310                 return;
 311             }
 312             createFibEntries(identifier, update);
 313             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 314                 rd, update.getDestPrefix(), update.getRoutePaths());
 315             return;
 316         }
 317
 318         /* Handl all other route origins */
 319         createFibEntries(identifier, update);
 320
 321         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 322             rd, update.getDestPrefix(), update.getRoutePaths());
 323     }
 324
 325     @Override
 326     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 327                        Object original, Object update, List<SubTransaction> transactionObjects) {
 328         this.transactionObjects = transactionObjects;
 329         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 330             createFibEntries(tx, identifier, (VrfEntry) update);
 331         }
 332     }
 333
 334     @Override
 335     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 336                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 337         this.transactionObjects = transactionObjects;
 338         if (vrfEntry instanceof VrfEntry) {
 339             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 340         }
 341     }
 342
 343     @Override
 344     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 345                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 346         this.transactionObjects = transactionObjects;
 347         if (vrfEntry instanceof VrfEntry) {
 348             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 349         }
 350     }
 351
 352     @Override
 353     public int getBatchSize() {
 354         return batchSize;
 355     }
 356
 357     @Override
 358     public int getBatchInterval() {
 359         return batchInterval;
 360     }
 361
 362     @Override
 363     public LogicalDatastoreType getDatastoreType() {
 364         return LogicalDatastoreType.CONFIGURATION;
 365     }
 366
 367     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 368         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 369
 370         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 371         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 372         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 373                 + " has null vpnId!");
 374         final Collection<VpnToDpnList> vpnToDpnList;
 375         if (vrfEntry.getParentVpnRd() != null
 376                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 377             VpnInstanceOpDataEntry parentVpnInstance = getVpnInstance(vrfEntry.getParentVpnRd());
 378             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 379                 vpnInstance.getVpnToDpnList();
 380         } else {
 381             vpnToDpnList = vpnInstance.getVpnToDpnList();
 382         }
 383
 384         final Long vpnId = vpnInstance.getVpnId();
 385         final String rd = vrfTableKey.getRouteDistinguisher();
 386         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 387         if (subnetRoute != null) {
 388             final long elanTag = subnetRoute.getElantag();
 389             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 390                     rd, vrfEntry.getDestPrefix(), elanTag);
 391             if (vpnToDpnList != null) {
 392                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 393                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
 394                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 395                     for (final VpnToDpnList curDpn : vpnToDpnList) {
 396                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 397                             installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 398                         }
 399                     }
 400                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 401                     futures.add(tx.submit());
 402                     return futures;
 403                 });
 404             }
 405             return;
 406         }
 407         // ping responder for router interfaces
 408         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 409             return;
 410         }
 411
 412         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 413         if (!localDpnIdList.isEmpty()) {
 414             if (vpnToDpnList != null) {
 415                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 416                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
 417                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 418                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 419                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 420                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 421                                 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry,
 422                                         tx);
 423                             }
 424                         }
 425                     }
 426                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 427                     futures.add(tx.submit());
 428                     return futures;
 429                 });
 430             }
 431         }
 432
 433         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 434         if (optVpnUuid.isPresent()) {
 435             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 436                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 437             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 438             if (optInterVpnLink.isPresent()) {
 439                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 440                 String vpnUuid = optVpnUuid.get();
 441                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 442                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 443                         // This is an static route that points to the other endpoint of an InterVpnLink
 444                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 445                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 446                         installInterVpnRouteInLFib(rd, vrfEntry);
 447                     }
 448                 });
 449             }
 450         }
 451     }
 452
 453     /*
 454       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 455       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 456       provided by ResourceBatchingManager
 457      */
 458     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 459                                   final VrfEntry vrfEntry) {
 460         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 461
 462         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 463         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 464         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 465             + " has null vpnId!");
 466
 467         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 468         if (vpnToDpnList != null) {
 469             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 470                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 471                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 472                 }
 473             }
 474         }
 475     }
 476
 477     // FIXME: Refactoring needed here.
 478     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 479     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 480                                    int addOrRemove) {
 481         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 482         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 483
 484         String prefix = vrfEntry.getDestPrefix();
 485         List<String> nextHopsList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 486         // Label is used only for logging in subsequent method calls.
 487         //TODO : This label is not needed here. Can be removed. Hence using a default value.
 488         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(0L);
 489         String rd = vrfTableKey.getRouteDistinguisher();
 490         LOG.trace("leakRouteIfNeeded: srcVpnRd={}  prefix={}  nhList={}  label={}", rd, prefix, nextHopsList, label);
 491
 492         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 493         if (vpnInstance == null) {
 494             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 495                       prefix, label, nextHopsList, rd);
 496             return;
 497         }
 498         String vpnUuid = vpnInstance.getVpnInstanceName();
 499         if (vpnUuid == null || vpnUuid.isEmpty()) {
 500             LOG.warn("Could not find suitable VPN UUID for rd={}.  vrfEntry=[prefix={}  nhList={}  label={}]",
 501                      rd, prefix, nextHopsList, label);
 502             return;
 503         }
 504
 505         ivpnLinkService.leakRouteIfNeeded(vpnUuid, prefix, nextHopsList, label.intValue(),
 506                                           RouteOrigin.value(vrfEntry.getOrigin()), addOrRemove);
 507     }
 508
 509     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 510         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 511         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 512         prefixBuilder.setDpnId(lri.getDpnId());
 513         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 514         prefixBuilder.setIpAddress(lri.getPrefix());
 515         // Increment the refCount here
 516         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 517             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 518         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 519         if (!isPresentInList) {
 520             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 521             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 522             vpnInstanceNames.add(vpnInstanceName);
 523             builder.setVpnInstanceList(vpnInstanceNames);
 524             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 525                 FibUtil.DEFAULT_CALLBACK);
 526         } else {
 527             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 528         }
 529         return prefixBuilder.build();
 530     }
 531
 532     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 533                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 534         Boolean wrTxPresent = true;
 535         if (tx == null) {
 536             wrTxPresent = false;
 537             tx = dataBroker.newWriteOnlyTransaction();
 538         }
 539         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 540             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 541             synchronized (label.toString().intern()) {
 542                 LabelRouteInfo lri = getLabelRouteInfo(label);
 543                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 544
 545                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 546                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 547                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 548                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 549                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 550                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 551                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 552                             }
 553                         }
 554                     }
 555                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 556                             label, lri.getVpnInterfaceName(), lri.getDpnId());
 557                 }
 558             }
 559         });
 560         final List<InstructionInfo> instructions = new ArrayList<>();
 561         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(24))
 562             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 563         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 564         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 565         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 566
 567         for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 568             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 569                 List<ActionInfo> actionsInfos = new ArrayList<>();
 570                 // reinitialize instructions list for LFIB Table
 571                 final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 572
 573                 actionsInfos.add(new ActionPopMpls());
 574                 LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 575                 LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 576                         MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 577                 LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 578
 579                 makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 580                         NwConstants.ADD_FLOW, tx);
 581             }
 582         }
 583         if (!wrTxPresent) {
 584             tx.submit();
 585         }
 586     }
 587
 588     /*
 589      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 590      * LportDispatcher table (via table 80)
 591      */
 592     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 593         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 594         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 595         // packet is commuted from Vpn2 to Vpn1.
 596         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 597         if (!vpnNameOpc.isPresent()) {
 598             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 599             return;
 600         }
 601
 602         String vpnName = vpnNameOpc.get();
 603         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 604         boolean interVpnLinkFound = false;
 605         for (InterVpnLink interVpnLink : interVpnLinks) {
 606             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 607             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 608                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 609             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 610                 interVpnLinkFound = true;
 611
 612                 Optional<InterVpnLinkState> vpnLinkState =
 613                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 614                 if (!vpnLinkState.isPresent()
 615                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 616                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 617                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 618                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 619                     return;
 620                 }
 621
 622                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 623                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 624                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 625                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 626
 627                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 628
 629                 for (BigInteger dpId : targetDpns) {
 630                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 631
 632                     List<InstructionInfo> instructions =
 633                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 634                             new InstructionWriteMetadata(
 635                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 636                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 637                                         NwConstants.L3VPN_SERVICE_INDEX)),
 638                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 639                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 640
 641                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(interVpnRoutePathLabel -> {
 642                         List<String> interVpnNextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 643                         LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 644                                 + "InterVpnLink {} in LFIB",
 645                                 vrfEntry.getDestPrefix(), interVpnRoutePathLabel, interVpnNextHopList,
 646                                 dpId, interVpnLink.getName());
 647
 648                         makeLFibTableEntry(dpId, interVpnRoutePathLabel, instructions, LFIB_INTERVPN_PRIORITY,
 649                                 NwConstants.ADD_FLOW, null);
 650                     });
 651                 }
 652
 653                 break;
 654             }
 655         }
 656
 657         if (!interVpnLinkFound) {
 658             LOG.warn("VrfEntry=[prefix={} route-paths={}] for VPN {} has origin INTERVPN but "
 659                 + "no InterVpnLink could be found",
 660                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), rd);
 661         }
 662     }
 663
 664
 665     /*
 666      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 667      */
 668     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 669                                                final VrfEntry vrfEntry, long vpnTag) {
 670         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 671         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 672             && vrfEntry.getRoutePaths().size() == 1);
 673         String destination = vrfEntry.getDestPrefix();
 674         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 675         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 676
 677         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 678         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 679         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 680         if (interVpnLink.getState().or(State.Error) != State.Active) {
 681             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 682                 destination, nextHop, interVpnLinkName);
 683             return;
 684         }
 685
 686         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 687         if (!optOtherEndpointLportTag.isPresent()) {
 688             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 689                 vpnUuid, interVpnLinkName);
 690             return;
 691         }
 692
 693         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 694         if (targetDpns.isEmpty()) {
 695             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 696                 vpnUuid, interVpnLinkName);
 697             return;
 698         }
 699
 700         String[] values = destination.split("/");
 701         String destPrefixIpAddress = values[0];
 702         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 703
 704         List<MatchInfo> matches = new ArrayList<>();
 705         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 706         matches.add(MatchEthernetType.IPV4);
 707
 708         if (prefixLength != 0) {
 709             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 710         }
 711
 712         List<Instruction> instructions =
 713             Arrays.asList(new InstructionWriteMetadata(
 714                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 715                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 716                             .L3VPN_SERVICE_INDEX)),
 717                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 718                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 719
 720         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 721         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 722         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 723             COOKIE_VM_FIB_TABLE, matches, instructions);
 724
 725         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 726             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 727
 728         for (BigInteger dpId : targetDpns) {
 729
 730             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 731                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 732                 dpId, interVpnLink.getInterVpnLinkName());
 733
 734             mdsalManager.installFlow(dpId, flowEntity);
 735         }
 736     }
 737
 738
 739     // TODO Clean up the exception handling
 740     @SuppressWarnings("checkstyle:IllegalCatch")
 741     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 742                                                     InstanceIdentifier<T> path) {
 743
 744         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 745
 746         Optional<T> result = Optional.absent();
 747         try {
 748             result = tx.read(datastoreType, path).get();
 749         } catch (Exception e) {
 750             throw new RuntimeException(e);
 751         }
 752
 753         return result;
 754     }
 755
 756     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 757         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 758         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 759
 760         if (localNextHopInfo == null) {
 761             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 762             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
 763                     FibUtil.getVpnNameFromId(dataBroker, vpnId), rd, vrfEntry.getDestPrefix());
 764             if (extraRouteOptional.isPresent()) {
 765                 Routes extraRoute = extraRouteOptional.get();
 766                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 767                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 768                     if (nextHopIp != null) {
 769                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp
 770                                 + NwConstants.IPV4PREFIX);
 771                         if (localNextHopInfo != null) {
 772                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 773                         }
 774                     }
 775                 }
 776             }
 777         } else {
 778             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 779         }
 780
 781         return returnLocalDpnId;
 782     }
 783
 784     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 785         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 786         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 787         String localNextHopIP = vrfEntry.getDestPrefix();
 788         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 789         if (localNextHopInfo == null) {
 790             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
 791             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 792                     vpnName, usedRds, localNextHopIP);
 793             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 794             vpnExtraRoutes.stream().forEach(extraRoute -> {
 795                 Prefixes localNextHopInfoLocal = FibUtil.getPrefixToInterface(dataBroker,
 796                         vpnId, extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
 797                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfoLocal, vrfEntry.getDestPrefix(),
 798                         vpnId, rd, vrfEntry, vpnId, extraRoute, vpnExtraRoutes);
 799                 returnLocalDpnId.add(dpnId);
 800             });
 801             if (localNextHopInfo == null) {
 802             /* imported routes case */
 803                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 804                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 805                     if (optionalLabel.isPresent()) {
 806                         Long label = optionalLabel.get();
 807                         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 808                         synchronized (label.toString().intern()) {
 809                             LabelRouteInfo lri = getLabelRouteInfo(label);
 810                             if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 811                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 812                                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 813                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
 814                                     String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 815                                     if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 816                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 817                                         localNextHopIP = lri.getPrefix();
 818                                     } else {
 819                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 820                                         localNextHopIP = lri.getPrefix();
 821                                     }
 822                                 }
 823                                 if (localNextHopInfo != null) {
 824                                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 825                                             label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 826                                     if (vpnExtraRoutes == null || vpnExtraRoutes.isEmpty()) {
 827                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 828                                                 vpnId, rd, vrfEntry, lri.getParentVpnid(), null, vpnExtraRoutes);
 829                                         returnLocalDpnId.add(dpnId);
 830                                     } else {
 831                                         for (Routes extraRoutes : vpnExtraRoutes) {
 832                                             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo,
 833                                                     localNextHopIP,
 834                                                     vpnId, rd, vrfEntry, lri.getParentVpnid(),
 835                                                     extraRoutes, vpnExtraRoutes);
 836                                             returnLocalDpnId.add(dpnId);
 837                                         }
 838                                     }
 839                                 }
 840                             }
 841                         }
 842                     }
 843                 }
 844             }
 845             if (returnLocalDpnId.isEmpty()) {
 846                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 847             }
 848         } else {
 849             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 850                     rd, vrfEntry, vpnId, /*routes*/ null, /*vpnExtraRoutes*/ null);
 851             returnLocalDpnId.add(dpnId);
 852         }
 853         return returnLocalDpnId;
 854     }
 855
 856     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 857                                                 final Long vpnId, final String rd,
 858                                                 final VrfEntry vrfEntry, Long parentVpnId,
 859                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 860         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 861         if (localNextHopInfo != null) {
 862             long groupId;
 863             long localGroupId;
 864             final BigInteger dpnId = localNextHopInfo.getDpnId();
 865             if (Boolean.TRUE.equals(localNextHopInfo.isNatPrefix())) {
 866                 LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip local dpn {} FIB processing",
 867                         vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 868                 return dpnId;
 869             }
 870             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 871             if (routes != null) {
 872                 groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 873                         vpnExtraRoutes);
 874                 localGroupId = nextHopManager.getLocalNextHopGroup(parentVpnId,
 875                         routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
 876             } else {
 877                 groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 878                         localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix(),
 879                         vrfEntry.getGatewayMacAddress(), jobKey);
 880                 localGroupId = groupId;
 881             }
 882             if (groupId == FibConstants.INVALID_GROUP_ID) {
 883                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 884                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 885                 return BigInteger.ZERO;
 886             }
 887             final List<InstructionInfo> instructions = Collections.singletonList(
 888                 new InstructionApplyActions(
 889                     Collections.singletonList(new ActionGroup(groupId))));
 890             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 891                 new InstructionApplyActions(
 892                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 893             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 894             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 895             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 896                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 897                     dpnId, localNextHopInfo.getVpnInterfaceName(), optLabel);
 898             } else {
 899                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 900                     + "LFib and Terminating table entries will not be created.",
 901                     rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 902             }
 903             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 904
 905             dataStoreCoordinator.enqueueJob(jobKey,
 906                 () -> {
 907                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 908                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 909                     optLabel.ifPresent(label -> {
 910                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 911                             makeLFibTableEntry(dpnId, label, lfibinstructions,
 912                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 913                             makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 914                         }
 915                     });
 916
 917                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 918                     futures.add(tx.submit());
 919                     return futures;
 920                 });
 921             return dpnId;
 922         }
 923         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}",
 924                 vrfEntry.getDestPrefix(), rd, vpnName);
 925         return BigInteger.ZERO;
 926     }
 927
 928     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 929         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 930         return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id).isPresent();
 931     }
 932
 933     private LabelRouteInfo getLabelRouteInfo(Long label) {
 934         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 935             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 936         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 937         if (opResult.isPresent()) {
 938             return opResult.get();
 939         }
 940         return null;
 941     }
 942
 943     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 944         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 945         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 946             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 947         if (lri == null) {
 948             return true;
 949         }
 950         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 951             ? lri.getVpnInstanceList() : new ArrayList<>();
 952         if (vpnInstancesList.contains(vpnInstanceName)) {
 953             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 954             vpnInstancesList.remove(vpnInstanceName);
 955         }
 956         if (vpnInstancesList.size() == 0) {
 957             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 958             if (tx != null) {
 959                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 960             } else {
 961                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 962             }
 963             return true;
 964         } else {
 965             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 966             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 967             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 968                 FibUtil.DEFAULT_CALLBACK);
 969         }
 970         return false;
 971     }
 972
 973     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 974                                       WriteTransaction tx) {
 975         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 976
 977         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 978
 979         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 980             dpId, label, groupId);
 981     }
 982
 983     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 984                                                 WriteTransaction tx) {
 985         List<MatchInfo> mkMatches = new ArrayList<>();
 986
 987         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 988             destDpId, label, actionsInfos);
 989
 990         // Matching metadata
 991         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 992         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 993
 994         List<InstructionInfo> mkInstructions = new ArrayList<>();
 995         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 996
 997         FlowEntity terminatingServiceTableFlowEntity =
 998             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
 999             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1000                 String.format("%s:%d", "TST Flow Entry ", label),
1001             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1002
1003         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1004
1005         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1006
1007         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1008         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1009             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1010             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1011             .child(Flow.class, flowKey).build();
1012         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
1013     }
1014
1015     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1016         FlowEntity flowEntity;
1017         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1018         List<MatchInfo> mkMatches = new ArrayList<>();
1019         // Matching metadata
1020         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1021         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1022             NwConstants.INTERNAL_TUNNEL_TABLE,
1023             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1024             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1025             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1026         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
1027         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1028         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1029             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1030             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1031
1032         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1033         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1034     }
1035
1036     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1037         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1038         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1039         String localNextHopIP = vrfEntry.getDestPrefix();
1040         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1041
1042         if (localNextHopInfo == null) {
1043             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1044             if (usedRds.size() > 1) {
1045                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1046                         vrfEntry.getDestPrefix(), vpnName, rd);
1047                 return returnLocalDpnId;
1048             }
1049             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1050             //in the vpn
1051             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1052                     vpnName, rd, vrfEntry.getDestPrefix());
1053             if (extraRouteOptional.isPresent()) {
1054                 Routes extraRoute = extraRouteOptional.get();
1055                 localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId,
1056                         extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
1057                 if (localNextHopInfo != null) {
1058                     BigInteger dpnId = localNextHopInfo.getDpnId();
1059                     if (!dpnId.equals(BigInteger.ZERO)) {
1060                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1061                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ null, /*remove*/ false);
1062                         returnLocalDpnId.add(dpnId);
1063                     }
1064                 }
1065             }
1066
1067             if (localNextHopInfo == null) {
1068                 /* Imported VRF entry */
1069                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1070                 if (optionalLabel.isPresent()) {
1071                     Long label = optionalLabel.get();
1072                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1073                     LabelRouteInfo lri = getLabelRouteInfo(label);
1074                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1075                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1076                         prefixBuilder.setDpnId(lri.getDpnId());
1077                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), localNextHopIP,
1078                                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1079                         if (!dpnId.equals(BigInteger.ZERO)) {
1080                             returnLocalDpnId.add(dpnId);
1081                         }
1082                     }
1083                 }
1084             }
1085
1086         } else {
1087             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1088                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1089             if (!dpnId.equals(BigInteger.ZERO)) {
1090                 returnLocalDpnId.add(dpnId);
1091             }
1092         }
1093
1094         return returnLocalDpnId;
1095     }
1096
1097     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1098                                                 final Long vpnId, final String rd,
1099                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1100         if (localNextHopInfo != null) {
1101             final BigInteger dpnId = localNextHopInfo.getDpnId();
1102             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1103             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1104                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1105                 () -> {
1106                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1107                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1108                         NwConstants.DEL_FLOW, tx);
1109                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1110                         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1111                             makeLFibTableEntry(dpnId, label, null /* instructions */,
1112                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1113                             removeTunnelTableEntry(dpnId, label, tx);
1114                         });
1115                     }
1116                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1117                     futures.add(tx.submit());
1118                     return futures;
1119                 });
1120             //TODO: verify below adjacency call need to be optimized (?)
1121             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1122             return dpnId;
1123         }
1124         return BigInteger.ZERO;
1125     }
1126
1127     static  InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
1128             String ipPrefix) {
1129         return InstanceIdentifier.builder(VpnToExtraroutes.class)
1130                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
1131                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
1132     }
1133
1134     public Routes getVpnToExtraroute(String vpnRd, String destPrefix) {
1135         Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, vpnRd);
1136         if (optVpnName.isPresent()) {
1137             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
1138                     optVpnName.get(), vpnRd, destPrefix);
1139             return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
1140         }
1141         return null;
1142     }
1143
1144     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1145             final VrfEntry vrfEntry, WriteTransaction tx) {
1146         Boolean wrTxPresent = true;
1147         if (tx == null) {
1148             wrTxPresent = false;
1149             tx = dataBroker.newWriteOnlyTransaction();
1150         }
1151         String rd = vrfTableKey.getRouteDistinguisher();
1152         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1153         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1154                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1155
1156         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1157         if (adjacencyResults == null || adjacencyResults.isEmpty()) {
1158             LOG.error("Could not get interface for route-paths: {} in vpn {}",
1159                     vrfEntry.getRoutePaths(), rd);
1160             LOG.warn("Failed to add Route: {} in vpn: {}",
1161                     vrfEntry.getDestPrefix(), rd);
1162             return;
1163         }
1164         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1165             programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1166         } else {
1167             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1168             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1169                     vpnName, usedRds, vrfEntry.getDestPrefix());
1170             if (!vpnExtraRoutes.isEmpty()) {
1171                 List<InstructionInfo> instructions = new ArrayList<>();
1172                 long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1173                         null, vpnExtraRoutes);
1174                 if (groupId == FibConstants.INVALID_GROUP_ID) {
1175                     LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1176                             vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1177                     return;
1178                 }
1179                 List<ActionInfo> actionInfos =
1180                         Collections.singletonList(new ActionGroup(groupId));
1181                 instructions.add(new InstructionApplyActions(actionInfos));
1182                 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1183             } else {
1184                 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1185             }
1186         }
1187         if (!wrTxPresent) {
1188             tx.submit();
1189         }
1190         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1191     }
1192
1193     private void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
1194             final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
1195         List<InstructionInfo> instructions = new ArrayList<>();
1196         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1197             List<ActionInfo> actionInfos = new ArrayList<>();
1198             String egressInterface = adjacencyResult.getInterfaceName();
1199             if (FibUtil.isTunnelInterface(adjacencyResult)) {
1200                 addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos);
1201             } else {
1202                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1203             }
1204             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
1205                     actionInfos.size());
1206             if (egressActions.isEmpty()) {
1207                 LOG.error(
1208                         "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
1209                                 + "Aborting remote FIB entry creation.",
1210                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1211                 return;
1212             }
1213             actionInfos.addAll(egressActions);
1214             instructions.add(new InstructionApplyActions(actionInfos));
1215         }
1216         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1217     }
1218
1219     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1220         if (vrfEntry.getMac() != null) {
1221             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1222                 new MacAddress(vrfEntry.getMac())));
1223             return;
1224         }
1225
1226         String ipPrefix = vrfEntry.getDestPrefix();
1227         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1228         if (prefixInfo == null) {
1229             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1230             return;
1231         }
1232
1233         String ifName = prefixInfo.getVpnInterfaceName();
1234         if (ifName == null) {
1235             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1236             return;
1237         }
1238
1239         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1240         if (macAddress == null) {
1241             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1242             return;
1243         }
1244
1245         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1246     }
1247
1248     private void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, long vpnId, VrfEntry vrfEntry,
1249                                            List<ActionInfo> actionInfos) {
1250         Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(interfaceManager,
1251                 adjacencyResult.getInterfaceName());
1252         // TODO - For now have added routePath into adjacencyResult so that we know for which
1253         // routePath this result is built for. If this is not possible construct a map which does
1254         // the same.
1255         String nextHopIp = adjacencyResult.getNextHopIp();
1256         java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
1257         if (!optionalLabel.isPresent()) {
1258             LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
1259             return;
1260         }
1261         long label = optionalLabel.get();
1262         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1263             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1264             actionInfos.add(new ActionPushMpls());
1265             actionInfos.add(new ActionSetFieldMplsLabel(label));
1266             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1267         } else {
1268             BigInteger tunnelId;
1269             // FIXME vxlan vni bit set is not working properly with OVS.need to
1270             // revisit
1271             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1272                 tunnelId = BigInteger.valueOf(label);
1273             } else {
1274                 tunnelId = BigInteger.valueOf(label);
1275             }
1276
1277             LOG.debug("adding set tunnel id action for label {}", label);
1278             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1279             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1280         }
1281     }
1282
1283     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1284         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1285         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1286         if (dpnInVpn.isPresent()) {
1287             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1288             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1289
1290             if (vpnInterfaces.remove(currVpnInterface)) {
1291                 if (vpnInterfaces.isEmpty()) {
1292                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1293                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1294                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1295                 } else {
1296                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1297                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1298                         VpnInterfaces.class,
1299                         new VpnInterfacesKey(intfName)));
1300                 }
1301             }
1302         }
1303     }
1304
1305     void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1306     /* Get interface info from prefix to interface mapping;
1307         Use the interface info to get the corresponding vpn interface op DS entry,
1308         remove the adjacency corresponding to this fib entry.
1309         If adjacency removed is the last adjacency, clean up the following:
1310          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1311          - prefix to interface entry
1312          - vpn interface op DS
1313      */
1314         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1315         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1316         Routes extraRoute = null;
1317         if (prefixInfo == null) {
1318             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1319             if (extraRoute != null) {
1320                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1321                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1322                     if (nextHopIp != null) {
1323                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp
1324                                 + NwConstants.IPV4PREFIX);
1325                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1326                     }
1327                 }
1328             }
1329             if (prefixInfo == null) {
1330                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1331                 if (optionalLabel.isPresent()) {
1332                     Long label = optionalLabel.get();
1333                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1334                     LabelRouteInfo lri = getLabelRouteInfo(label);
1335                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1336                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1337                         prefixBuilder.setDpnId(lri.getDpnId());
1338                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1339                         prefixBuilder.setIpAddress(lri.getPrefix());
1340                         prefixInfo = prefixBuilder.build();
1341                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1342                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1343                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1344                     }
1345                 }
1346             }
1347         } else {
1348             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1349         }
1350     }
1351
1352     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1353                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1354
1355         if (prefixInfo == null) {
1356             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1357             return; //Don't have any info for this prefix (shouldn't happen); need to return
1358         }
1359
1360         if (Boolean.TRUE.equals(prefixInfo.isNatPrefix())) {
1361             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1362                     vrfEntry.getDestPrefix(), vpnId, rd);
1363             return;
1364         }
1365
1366         String ifName = prefixInfo.getVpnInterfaceName();
1367         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1368         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1369             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1370     }
1371
1372     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1373         Prefixes prefixInfo;
1374         Long vpnId;
1375         String rd;
1376         VrfEntry vrfEntry;
1377         Routes extraRoute;
1378
1379         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1380                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1381             this.prefixInfo = prefixInfo;
1382             this.vpnId = vpnId;
1383             this.rd = rd;
1384             this.vrfEntry = vrfEntry;
1385             this.extraRoute = extraRoute;
1386         }
1387
1388         @Override
1389         public List<ListenableFuture<Void>> call() throws Exception {
1390             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1391             // to call the respective helpers.
1392             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1393
1394             //First Cleanup LabelRouteInfo
1395             //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1396             if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Mplsgre)) {
1397                 FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1398                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1399                     synchronized (label.toString().intern()) {
1400                         LabelRouteInfo lri = getLabelRouteInfo(label);
1401                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1402                                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1403                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1404                                     FibUtil.getVpnInstanceOpData(dataBroker, rd);
1405                             String vpnInstanceName = "";
1406                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
1407                                 vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1408                             }
1409                             boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, writeOperTxn);
1410                             if (lriRemoved) {
1411                                 String parentRd = lri.getParentVpnRd();
1412                                 FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1413                                         FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1414                             }
1415                         } else {
1416                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1417                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1418                         }
1419                     }
1420                 });
1421             }
1422             String ifName = prefixInfo.getVpnInterfaceName();
1423             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1424                 FibUtil.getVpnInterfaceIdentifier(ifName));
1425             if (optvpnInterface.isPresent()) {
1426                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1427                 if (vpnId != associatedVpnId) {
1428                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1429                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1430                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1431                     return null;
1432                 } else {
1433                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1434                         vrfEntry.getDestPrefix(), associatedVpnId);
1435                 }
1436             }
1437             if (extraRoute != null) {
1438                 Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, rd);
1439                 if (optVpnName.isPresent()) {
1440                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1441                             getVpnToExtrarouteIdentifier(optVpnName.get(), rd, vrfEntry.getDestPrefix()));
1442                 }
1443             }
1444             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1445                 FibUtil.getAdjListPath(ifName));
1446             int numAdj = 0;
1447             if (optAdjacencies.isPresent()) {
1448                 numAdj = optAdjacencies.get().getAdjacency().size();
1449             }
1450             //remove adjacency corr to prefix
1451             if (numAdj > 1) {
1452                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId,
1453                         vrfEntry.getDestPrefix());
1454                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1455                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1456             } else {
1457                 //this is last adjacency (or) no more adjacency left for this vpn interface, so
1458                 //clean up the vpn interface from DpnToVpn list
1459                 LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1460                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName));
1461             }
1462             List<ListenableFuture<Void>> futures = new ArrayList<>();
1463             futures.add(writeOperTxn.submit());
1464             return futures;
1465         }
1466     }
1467
1468     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1469         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1470         final String rd = vrfTableKey.getRouteDistinguisher();
1471         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1472         if (vpnInstance == null) {
1473             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1474             return;
1475         }
1476         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1477         long elanTag = 0L;
1478         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1479         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1480         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1481         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1482         if (subnetRoute != null) {
1483             elanTag = subnetRoute.getElantag();
1484             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1485                 rd, vrfEntry.getDestPrefix(), elanTag);
1486             if (vpnToDpnList != null) {
1487                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1488                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
1489                     () -> {
1490                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1491
1492                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1493
1494                             makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1495                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1496                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1497                                 optionalLabel.ifPresent(label -> {
1498                                     makeLFibTableEntry(curDpn.getDpnId(), label, null,
1499                                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1500                                 });
1501                             }
1502                         }
1503                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1504                         futures.add(tx.submit());
1505                         return futures;
1506                     });
1507             }
1508             optionalLabel.ifPresent(label -> {
1509                 synchronized (label.toString().intern()) {
1510                     LabelRouteInfo lri = getLabelRouteInfo(label);
1511                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1512                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1513                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
1514                         String vpnInstanceName = "";
1515                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1516                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1517                         }
1518                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1519                         if (lriRemoved) {
1520                             String parentRd = lri.getParentVpnRd();
1521                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1522                                     FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1523                             LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1524                                     + "labelRouteInfo cleared", label, rd,
1525                                     vrfEntry.getDestPrefix());
1526                         }
1527                     } else {
1528                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1529                                 FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1530                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1531                                 label, rd, vrfEntry.getDestPrefix());
1532                     }
1533                 }
1534             });
1535             return;
1536         }
1537         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1538             return;
1539         }
1540
1541         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1542             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1543         if (vpnToDpnList != null) {
1544             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1545                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1546             String usedRd = null;
1547             Optional<Routes> extraRouteOptional;
1548             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1549             if (usedRds != null && !usedRds.isEmpty()) {
1550                 if (usedRds.size() > 1) {
1551                     LOG.error("The extra route prefix is still present in some DPNs");
1552                     return ;
1553                 } else {
1554                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1555                     //is not present in any other DPN
1556                     extraRouteOptional = VpnExtraRouteHelper
1557                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1558                 }
1559             } else {
1560                 extraRouteOptional = Optional.absent();
1561             }
1562             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1563             dataStoreCoordinator.enqueueJob("FIB-" + usedRd + "-" + vrfEntry.getDestPrefix(),
1564                 () -> {
1565                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1566
1567                     if (localDpnIdList.size() <= 0) {
1568                         for (VpnToDpnList curDpn : vpnToDpnList) {
1569                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1570                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1571                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1572                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1573                                 }
1574                             } else {
1575                                 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1576                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1577                             }
1578                         }
1579                     } else {
1580                         for (BigInteger localDpnId : localDpnIdList) {
1581                             for (VpnToDpnList curDpn : vpnToDpnList) {
1582                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1583                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1584                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1585                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1586                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1587                                         }
1588                                     } else {
1589                                         deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1590                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1591                                     }
1592                                 }
1593                             }
1594                         }
1595                     }
1596                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1597                     futures.add(tx.submit());
1598                     return futures;
1599                 });
1600         }
1601
1602         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1603         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1604         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1605
1606         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1607         // of the interVpnLink.
1608         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1609         if (optVpnUuid.isPresent()) {
1610             String vpnUuid = optVpnUuid.get();
1611             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1612                 Optional<InterVpnLinkDataComposite> optInterVpnLink =
1613                         InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1614                 if (optInterVpnLink.isPresent()) {
1615                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1616                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1617                         // This is route that points to the other endpoint of an InterVpnLink
1618                         // In that case, we should look for the FIB table pointing to
1619                         // LPortDispatcher table and remove it.
1620                         removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1621                                 interVpnLink.isFirstEndpointVpnName(rd),
1622                                 vrfEntry);
1623                     }
1624                 }
1625             });
1626         }
1627
1628     }
1629
1630     /*
1631       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1632       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1633       provided by ResourceBatchingManager
1634      */
1635     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1636                                   final VrfEntry vrfEntry) {
1637         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1638         String rd = vrfTableKey.getRouteDistinguisher();
1639         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1640         if (vpnInstance == null) {
1641             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1642             return;
1643         }
1644         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1645         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1646         if (vpnToDpnList != null) {
1647             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1648                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1649             Optional<Routes> extraRouteOptional;
1650             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1651             if (usedRds != null && !usedRds.isEmpty()) {
1652                 if (usedRds.size() > 1) {
1653                     LOG.error("The extra route prefix is still present in some DPNs");
1654                     return ;
1655                 } else {
1656                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1657                             usedRds.get(0), vrfEntry.getDestPrefix());
1658                 }
1659             } else {
1660                 extraRouteOptional = Optional.absent();
1661             }
1662             for (VpnToDpnList curDpn : vpnToDpnList) {
1663                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1664                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey,
1665                             vrfEntry, extraRouteOptional, writeTx);
1666                 }
1667             }
1668         }
1669     }
1670
1671     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1672             final long vpnId, final VrfTablesKey vrfTableKey,
1673             final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional, WriteTransaction tx) {
1674
1675         Boolean wrTxPresent = true;
1676         if (tx == null) {
1677             wrTxPresent = false;
1678             tx = dataBroker.newWriteOnlyTransaction();
1679         }
1680
1681         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1682                 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1683         String rd = vrfTableKey.getRouteDistinguisher();
1684
1685         if (localDpnId != null && localDpnId != BigInteger.ZERO) {
1686             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1687             if (extraRouteOptional.isPresent()) {
1688                 nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
1689             }
1690             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1691             return;
1692         }
1693
1694         // below two reads are kept as is, until best way is found to identify dpnID
1695         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1696         if (extraRouteOptional.isPresent()) {
1697             nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
1698         } else {
1699             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1700         }
1701         if (!wrTxPresent) {
1702             tx.submit();
1703         }
1704     }
1705
1706     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1707                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1708         boolean isRemoteRoute = true;
1709         if (localNextHopInfo != null) {
1710             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1711         }
1712         if (isRemoteRoute) {
1713             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1714             return true;
1715         } else {
1716             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1717                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1718             return false;
1719         }
1720     }
1721
1722     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
1723             String rd, WriteTransaction tx) {
1724         // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
1725         // the adjacencyResults.
1726         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1727             List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1728             if (!adjacencyResults.isEmpty()) {
1729                 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1730                 return;
1731             }
1732         }
1733         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1734         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1735     }
1736
1737     private long get(byte[] rawIpAddress) {
1738         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1739             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1740     }
1741
1742     void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1743                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1744         Boolean wrTxPresent = true;
1745         if (tx == null) {
1746             wrTxPresent = false;
1747             tx = dataBroker.newWriteOnlyTransaction();
1748         }
1749
1750         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1751         String[] values = vrfEntry.getDestPrefix().split("/");
1752         String ipAddress = values[0];
1753         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1754         if (addOrRemove == NwConstants.ADD_FLOW) {
1755             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1756         } else {
1757             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1758         }
1759         InetAddress destPrefix;
1760         try {
1761             destPrefix = InetAddress.getByName(ipAddress);
1762         } catch (UnknownHostException e) {
1763             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1764             return;
1765         }
1766
1767         List<MatchInfo> matches = new ArrayList<>();
1768
1769         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1770
1771         if (destPrefix instanceof Inet4Address) {
1772             matches.add(MatchEthernetType.IPV4);
1773             if (prefixLength != 0) {
1774                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1775             }
1776         } else {
1777             matches.add(MatchEthernetType.IPV6);
1778             if (prefixLength != 0) {
1779                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1780             }
1781         }
1782
1783         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1784         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1785         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1786             flowRef, 0, 0,
1787             COOKIE_VM_FIB_TABLE, matches, instructions);
1788         Flow flow = flowEntity.getFlowBuilder().build();
1789         String flowId = flowEntity.getFlowId();
1790         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1791         Node nodeDpn = buildDpnNode(dpId);
1792
1793         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1794             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1795             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1796
1797         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1798             SubTransaction subTransaction = new SubTransactionImpl();
1799             if (addOrRemove == NwConstants.ADD_FLOW) {
1800                 subTransaction.setInstanceIdentifier(flowInstanceId);
1801                 subTransaction.setInstance(flow);
1802                 subTransaction.setAction(SubTransaction.CREATE);
1803             } else {
1804                 subTransaction.setInstanceIdentifier(flowInstanceId);
1805                 subTransaction.setAction(SubTransaction.DELETE);
1806             }
1807             transactionObjects.add(subTransaction);
1808         }
1809
1810         if (addOrRemove == NwConstants.ADD_FLOW) {
1811             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1812         } else {
1813             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1814         }
1815
1816         if (!wrTxPresent) {
1817             tx.submit();
1818         }
1819     }
1820
1821     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1822     Node buildDpnNode(BigInteger dpnId) {
1823         NodeId nodeId = new NodeId("openflow:" + dpnId);
1824         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1825
1826         return nodeDpn;
1827     }
1828
1829     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1830                                     int addOrRemove, WriteTransaction tx) {
1831         Boolean wrTxPresent = true;
1832         if (tx == null) {
1833             wrTxPresent = false;
1834             tx = dataBroker.newWriteOnlyTransaction();
1835         }
1836
1837         List<MatchInfo> matches = new ArrayList<>();
1838         matches.add(MatchEthernetType.MPLS_UNICAST);
1839         matches.add(new MatchMplsLabel(label));
1840
1841         // Install the flow entry in L3_LFIB_TABLE
1842         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1843
1844         FlowEntity flowEntity;
1845         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1846             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1847         Flow flow = flowEntity.getFlowBuilder().build();
1848         String flowId = flowEntity.getFlowId();
1849         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1850         Node nodeDpn = buildDpnNode(dpId);
1851         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1852             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1853             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1854
1855         if (addOrRemove == NwConstants.ADD_FLOW) {
1856             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1857         } else {
1858             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1859         }
1860         if (!wrTxPresent) {
1861             tx.submit();
1862         }
1863
1864         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1865             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1866     }
1867
1868     void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1869                                       final String ipPrefixAddress) {
1870         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1871         try {
1872             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1873         } catch (NullPointerException e) {
1874             LOG.trace("", e);
1875         }
1876     }
1877
1878     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1879                                     final FutureCallback<List<Void>> callback) {
1880         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1881         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1882         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1883         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1884         if (!vrfTable.isPresent()) {
1885             LOG.warn("VRF Table not yet available for RD {}", rd);
1886             if (callback != null) {
1887                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1888                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1889                 Futures.addCallback(listenableFuture, callback);
1890             }
1891             return;
1892         }
1893         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1894         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1895             () -> {
1896                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1897                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1898                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1899                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1900                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1901                         if (subnetRoute != null) {
1902                             long elanTag = subnetRoute.getElantag();
1903                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1904                             continue;
1905                         }
1906                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1907                         if (routerInt != null) {
1908                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1909                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1910                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1911                                 routerInt.getIpAddress(),
1912                                 new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1913                             continue;
1914                         }
1915                         //Handle local flow creation for imports
1916                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1917                             java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1918                             if (optionalLabel.isPresent()) {
1919                                 List<String> nextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1920                                 LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1921                                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1922                                     if (lri.getDpnId().equals(dpnId)) {
1923                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1924                                         continue;
1925                                     }
1926                                 }
1927                             }
1928                         }
1929
1930                         boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1931                                 vrfEntry, dpnId);
1932                         if (shouldCreateRemoteFibEntry) {
1933                             LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}",
1934                                     vrfEntry, dpnId);
1935                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1936                         }
1937                     }
1938                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1939                     futures.add(tx.submit());
1940                 }
1941                 if (callback != null) {
1942                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1943                     Futures.addCallback(listenableFuture, callback);
1944                 }
1945                 return futures;
1946             });
1947     }
1948
1949     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1950                                             final String localNextHopIp, final String remoteNextHopIp) {
1951         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1952             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1953         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1954         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1955         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1956         if (vrfTable.isPresent()) {
1957             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1958             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1959                 () -> {
1960                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1961                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1962                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1963                         vrfTable.get().getVrfEntry().stream()
1964                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1965                             .forEach(
1966                                 getConsumerForCreatingRemoteFib(dpnId, vpnId,
1967                                         rd, remoteNextHopIp, vrfTable,
1968                                         writeCfgTxn));
1969                         futures.add(writeCfgTxn.submit());
1970                     }
1971                     return futures;
1972                 });
1973         }
1974     }
1975
1976     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1977                                             final String localNextHopIp, final String remoteNextHopIp) {
1978         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1979             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1980         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1981         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1982         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1983         if (vrfTable.isPresent()) {
1984             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1985             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1986                 () -> {
1987                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1988                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1989                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1990                         // Handle Vpn Interface driven Routes only (i.e., STATIC and LOCAL)
1991                         vrfTable.get().getVrfEntry().stream()
1992                             .filter(vrfEntry -> {
1993                                 /* Ignore SubnetRoute entry */
1994                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1995                                         vrfEntry.getOrigin())));
1996                             })
1997                             .forEach(getConsumerForCreatingRemoteFib(dpnId, vpnId,
1998                                        rd, remoteNextHopIp, vrfTable,
1999                                        writeCfgTxn));
2000                         futures.add(writeCfgTxn.submit());
2001                     }
2002                     return futures;
2003                 });
2004         }
2005     }
2006
2007     public void manageRemoteRouteOnDPN(final boolean action,
2008                                        final BigInteger localDpnId,
2009                                        final long vpnId,
2010                                        final String rd,
2011                                        final String destPrefix,
2012                                        final String destTepIp,
2013                                        final long label) {
2014         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2015
2016         if (vpnInstance == null) {
2017             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
2018             return;
2019         }
2020         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2021         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
2022             () -> {
2023                 List<ListenableFuture<Void>> futures = new ArrayList<>();
2024                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
2025                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2026                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
2027                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
2028                     if (vrfEntry == null) {
2029                         return futures;
2030                     }
2031                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
2032                         action, localDpnId, vpnId, rd, destPrefix);
2033                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
2034                     VrfEntry modVrfEntry;
2035                     if (routePathList == null || (routePathList.isEmpty())) {
2036                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
2037                                 Collections.singletonList(destTepIp),
2038                                 RouteOrigin.value(vrfEntry.getOrigin()))
2039                                 .build();
2040                     } else {
2041                         modVrfEntry = vrfEntry;
2042                     }
2043
2044                     if (action == true) {
2045                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
2046                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
2047                     } else {
2048                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
2049                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
2050                                 vrfEntry.getDestPrefix());
2051                         if (usedRds.size() > 1) {
2052                             LOG.debug("The extra route prefix is still present in some DPNs");
2053                             return futures;
2054                         }
2055                         //Is this fib route an extra route? If yes, get the nexthop which would be
2056                         //an adjacency in the vpn
2057                         Optional<Routes> extraRouteOptional = Optional.absent();
2058                         if (usedRds.size() != 0) {
2059                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
2060                                     FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId()),
2061                                     usedRds.get(0), vrfEntry.getDestPrefix());
2062                         }
2063                         deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
2064                                 extraRouteOptional, writeTransaction);
2065                     }
2066                     futures.add(writeTransaction.submit());
2067                 }
2068                 return futures;
2069             });
2070     }
2071
2072     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
2073                                  final FutureCallback<List<Void>> callback) {
2074         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
2075         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2076         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2077         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2078         if (vrfTable.isPresent()) {
2079             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2080             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2081                 () -> {
2082                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2083                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2084                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2085                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2086                                 /* Handle subnet routes here */
2087                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2088                             if (subnetRoute != null) {
2089                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
2090                                         vrfEntry.getDestPrefix(),
2091                                         dpnId, rd);
2092                                 makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
2093                                 java.util.Optional.ofNullable(vrfEntry.getRoutePaths()).ifPresent(routePaths -> {
2094                                     routePaths.stream().forEach(routePath -> {
2095                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
2096                                                 DEFAULT_FIB_FLOW_PRIORITY,
2097                                                 NwConstants.DEL_FLOW, tx);
2098                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} "
2099                                                 + "for rd {} prefix {}",
2100                                                 routePath.getLabel(), rd,
2101                                                 vrfEntry.getDestPrefix());
2102                                     });
2103                                 });
2104                                 continue;
2105                             }
2106                             // ping responder for router interfaces
2107                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2108                             if (routerInt != null) {
2109                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
2110                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
2111                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
2112                                     routerInt.getIpAddress(),
2113                                     new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
2114                                 continue;
2115                             }
2116                             // Passing null as we don't know the dpn
2117                             // to which prefix is attached at this point
2118                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
2119                                     vrfEntry.getDestPrefix());
2120                             String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
2121                             Optional<Routes> extraRouteOptional;
2122                             //Is this fib route an extra route? If yes, get the nexthop which would be
2123                             //an adjacency in the vpn
2124                             if (usedRds != null && !usedRds.isEmpty()) {
2125                                 if (usedRds.size() > 1) {
2126                                     LOG.error("The extra route prefix is still present in some DPNs");
2127                                     return futures;
2128                                 } else {
2129                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
2130                                             usedRds.get(0), vrfEntry.getDestPrefix());
2131
2132                                 }
2133                             } else {
2134                                 extraRouteOptional = Optional.absent();
2135                             }
2136                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
2137                                     extraRouteOptional, tx);
2138                         }
2139                         futures.add(tx.submit());
2140                         if (callback != null) {
2141                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
2142                             Futures.addCallback(listenableFuture, callback);
2143                         }
2144                     }
2145                     return futures;
2146                 });
2147         }
2148     }
2149
2150     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2151                                            final String localNextHopIp, final String remoteNextHopIp) {
2152         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2153                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2154             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2155         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2156         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2157         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2158         if (vrfTable.isPresent()) {
2159             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2160             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2161                 () -> {
2162                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2163                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2164                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2165                         vrfTable.get().getVrfEntry().stream()
2166                             .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2167                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
2168                                 remoteNextHopIp, vrfTable, writeCfgTxn));
2169                         futures.add(writeCfgTxn.submit());
2170                     }
2171                     return futures;
2172                 });
2173         }
2174     }
2175
2176     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2177                                            final String localNextHopIp, final String remoteNextHopIp) {
2178         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2179                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2180             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2181         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2182         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2183         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2184         if (vrfTable.isPresent()) {
2185             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2186             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2187                 () -> {
2188                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2189                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2190                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2191                         vrfTable.get().getVrfEntry().stream()
2192                             .filter(vrfEntry -> {
2193                                 /* Ignore SubnetRoute entry */
2194                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(
2195                                         RouteOrigin.value(vrfEntry.getOrigin())));
2196                             })
2197                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId,
2198                                 rd, remoteNextHopIp, vrfTable, writeCfgTxn));
2199                         futures.add(writeCfgTxn.submit());
2200                     }
2201                     return futures;
2202                 });
2203         }
2204     }
2205
2206     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2207         InstanceIdentifierBuilder<VrfTables> idBuilder =
2208             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2209         InstanceIdentifier<VrfTables> id = idBuilder.build();
2210         return id;
2211     }
2212
2213     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2214         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + label
2215                 + NwConstants.FLOWID_SEPARATOR + priority;
2216     }
2217
2218     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2219         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + rd
2220                 + NwConstants.FLOWID_SEPARATOR + priority + NwConstants.FLOWID_SEPARATOR + destPrefix.getHostAddress();
2221     }
2222
2223     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2224         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2225                 .FLOWID_SEPARATOR + nextHop;
2226     }
2227
2228     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2229                                                      final VrfEntry vrfEntry, String rd) {
2230         List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2231         FibHelper.sortIpAddress(routePaths);
2232         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2233         List<String> prefixIpList = new ArrayList<>();
2234         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2235             remoteDpnId, vpnId, vrfEntry);
2236         try {
2237             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2238                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
2239                 List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
2240                         FibUtil.getVpnNameFromId(dataBroker, vpnId), usedRds, vrfEntry.getDestPrefix());
2241                 if (vpnExtraRoutes.isEmpty()) {
2242                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2243                 } else {
2244                     List<String> prefixIpListLocal = new ArrayList<>();
2245                     vpnExtraRoutes.stream().forEach(route -> {
2246                         route.getNexthopIpList().stream().forEach(extraRouteIp -> {
2247                             prefixIpListLocal.add(extraRouteIp + NwConstants.IPV4PREFIX);
2248                         });
2249                     });
2250                     prefixIpList = prefixIpListLocal;
2251                 }
2252             } else {
2253                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2254             }
2255
2256             for (String prefixIp : prefixIpList) {
2257                 if (routePaths == null || routePaths.isEmpty()) {
2258                     LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
2259                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2260                           prefixIp, null);
2261                     addAdjacencyResultToList(adjacencyList, adjacencyResult);
2262                     continue;
2263                 }
2264                 adjacencyList.addAll(routePaths.stream()
2265                         .map(routePath -> {
2266                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
2267                                     routePath.getNexthopAddress());
2268                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2269                                     prefixIp, routePath.getNexthopAddress());
2270                         })
2271                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
2272                         .distinct()
2273                         .collect(toList()));
2274             }
2275         } catch (NullPointerException e) {
2276             LOG.trace("", e);
2277         }
2278         return adjacencyList;
2279     }
2280
2281     private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
2282         if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2283             adjacencyList.add(adjacencyResult);
2284         }
2285     }
2286
2287     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2288         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2289             InstanceIdentifier.create(VpnInstanceOpData.class)
2290                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2291         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2292             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2293         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2294     }
2295
2296     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2297         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2298                 + tableMiss + FLOWID_PREFIX;
2299     }
2300
2301     /*
2302      * Install flow entry in protocol table to forward mpls
2303      * coming through gre tunnel to LFIB table.
2304      */
2305     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2306         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2307         // Instruction to goto L3 InterfaceTable
2308         List<InstructionInfo> instructions = new ArrayList<>();
2309         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2310         List<MatchInfo> matches = new ArrayList<>();
2311         matches.add(MatchEthernetType.MPLS_UNICAST);
2312         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2313             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2314                 NwConstants.L3_LFIB_TABLE),
2315             DEFAULT_FIB_FLOW_PRIORITY,
2316             "Protocol Table For LFIB",
2317             0, 0,
2318             cookieProtocolTable,
2319             matches, instructions);
2320
2321         if (addOrRemove == NwConstants.ADD_FLOW) {
2322             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2323             mdsalManager.installFlow(flowEntityToLfib);
2324         } else {
2325             mdsalManager.removeFlow(flowEntityToLfib);
2326         }
2327     }
2328
2329     public List<String> printFibEntries() {
2330         List<String> result = new ArrayList<>();
2331         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2332         result.add("-------------------------------------------------------------------");
2333         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2334         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2335         if (fibEntries.isPresent()) {
2336             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2337             for (VrfTables vrfTable : vrfTables) {
2338                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2339                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2340                     for (RoutePaths routePath : routePaths) {
2341                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2342                             vrfTable.getRouteDistinguisher(),
2343                             vrfEntry.getDestPrefix(), routePath.getNexthopAddress(),
2344                             routePath.getLabel(), vrfEntry.getOrigin()));
2345                     }
2346                     if (routePaths.isEmpty()) {
2347                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s",
2348                             vrfTable.getRouteDistinguisher(),
2349                             vrfEntry.getDestPrefix(), "local", vrfEntry.getOrigin()));
2350                     }
2351                 }
2352             }
2353         }
2354         return result;
2355     }
2356
2357
2358     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2359         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2360             .child(VrfTables.class, new VrfTablesKey(rd))
2361             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2362         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2363         if (vrfEntry.isPresent()) {
2364             return vrfEntry.get();
2365         }
2366         return null;
2367     }
2368
2369     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2370         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2371             .child(VrfTables.class, new VrfTablesKey(rd))
2372             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2373         return vrfEntryId;
2374     }
2375
2376     protected Boolean isIpv4Address(String ipAddress) {
2377         try {
2378             InetAddress address = InetAddress.getByName(ipAddress);
2379             if (address instanceof Inet4Address) {
2380                 return true;
2381             }
2382         } catch (UnknownHostException e) {
2383             LOG.warn("Invalid ip address {}", ipAddress, e);
2384             return false;
2385         }
2386         return false;
2387     }
2388
2389     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2390                                               long vpnId, int addOrRemove) {
2391         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2392         if (routerInt == null) {
2393             return false;
2394         }
2395         if (vpnToDpnList != null) {
2396             String routerId = routerInt.getUuid();
2397             String macAddress = routerInt.getMacAddress();
2398             String ipValue = routerInt.getIpAddress();
2399             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2400                     routerId, ipValue, macAddress);
2401             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2402                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2403                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2404                             new MacAddress(macAddress), addOrRemove);
2405                 }
2406             }
2407         }
2408         return true;
2409     }
2410
2411     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2412                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2413         String[] subSplit = routerInternalIp.split("/");
2414         if (!isIpv4Address(subSplit[0])) {
2415             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2416             return;
2417         }
2418
2419         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2420         if (!optionalLabel.isPresent()) {
2421             LOG.warn("Routes paths not present. Exiting installRouterFibEntry");
2422             return;
2423         }
2424         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2425         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2426             dpnId, routerInternalIp, vpnId, subSplit[0]);
2427
2428         List<MatchInfo> matches = new ArrayList<>();
2429
2430         matches.add(MatchIpProtocol.ICMP);
2431         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2432         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2433         matches.add(MatchEthernetType.IPV4);
2434         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2435
2436         List<ActionInfo> actionsInfos = new ArrayList<>();
2437
2438         // Set Eth Src and Eth Dst
2439         actionsInfos.add(new ActionMoveSourceDestinationEth());
2440         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2441
2442         // Move Ip Src to Ip Dst
2443         actionsInfos.add(new ActionMoveSourceDestinationIp());
2444         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2445
2446         // Set the ICMP type to 0 (echo reply)
2447         actionsInfos.add(new ActionSetIcmpType((short) 0));
2448
2449         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2450
2451         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2452
2453         List<InstructionInfo> instructions = new ArrayList<>();
2454
2455         instructions.add(new InstructionApplyActions(actionsInfos));
2456
2457         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2458         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, optionalLabel.get(),
2459                 priority);
2460
2461         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2462             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2463
2464         if (addOrRemove == NwConstants.ADD_FLOW) {
2465             mdsalManager.installFlow(flowEntity);
2466         } else {
2467             mdsalManager.removeFlow(flowEntity);
2468         }
2469     }
2470
2471     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2472                                              final boolean isVpnFirstEndPoint,
2473                                              final VrfEntry vrfEntry) {
2474         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
2475             && vrfEntry.getRoutePaths().size() == 1);
2476         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2477
2478         if (!interVpnLinkState.isPresent()) {
2479             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2480             return;
2481         }
2482
2483         List<BigInteger> targetDpns =
2484             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2485                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2486
2487         java.util.Optional<String> optionalNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2488         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2489
2490         // delete from FIB
2491         //
2492         optionalNextHop.ifPresent(nextHop -> {
2493             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2494             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2495             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
2496                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2497
2498             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2499                     interVpnLinkName, flowRef);
2500
2501             for (BigInteger dpId : targetDpns) {
2502                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2503                         vrfEntry.getDestPrefix(), nextHop,
2504                         dpId, interVpnLinkName);
2505
2506                 mdsalManager.removeFlow(dpId, flow);
2507             }
2508         });
2509
2510         // delete from LFIB
2511         //
2512         optionalLabel.ifPresent(label -> {
2513             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2514
2515             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2516             for (BigInteger dpId : targetDpns) {
2517                 LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2518                         vrfEntry.getDestPrefix(), label,
2519                         dpId, interVpnLinkName);
2520                 makeLFibTableEntry(dpId, label, null /* no instructions */,
2521                         LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2522             }
2523             tx.submit();
2524         });
2525     }
2526
2527     private Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
2528             final BigInteger dpnId, final long vpnId, final String rd,
2529             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2530             WriteTransaction writeCfgTxn) {
2531         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2532                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2533                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2534                 .findFirst()
2535                 .ifPresent(routes -> {
2536                     LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
2537                             vrfEntry.getDestPrefix(), rd, dpnId);
2538                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
2539                 });
2540     }
2541
2542     private Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
2543             final BigInteger dpnId, final long vpnId, final String rd,
2544             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2545             WriteTransaction writeCfgTxn) {
2546         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2547                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2548                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2549                 .findFirst()
2550                 .ifPresent(routes -> {
2551                     LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2552                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
2553                             Optional.absent(), writeCfgTxn);
2554                 });
2555     }
2556
2557     private boolean isPrefixAndNextHopPresentInLri(String prefix,
2558             List<String> nextHopAddressList, LabelRouteInfo lri) {
2559         return lri != null && lri.getPrefix().equals(prefix)
2560                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2561     }
2562
2563     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2564         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2565             return true;
2566         }
2567
2568         Prefixes prefix = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
2569         if (prefix != null) {
2570             BigInteger prefixDpnId = prefix.getDpnId();
2571             if (prefixDpnId == dpnId) {
2572                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2573                         vrfEntry, dpnId);
2574                 return false;
2575             }
2576         }
2577
2578         return true;
2579     }
2580
2581     private void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId, final long vpnId,
2582             final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
2583         Preconditions.checkArgument(vrfEntry.getRoutePaths().size() <= 2);
2584
2585         if (adjacencyResults.size() == 1) {
2586             programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
2587             return;
2588         }
2589         // ECMP Use case, point to LB group. Move the mpls label accordingly.
2590         List<String> tunnelList =
2591                 adjacencyResults.stream()
2592                 .map(adjacencyResult -> adjacencyResult.getNextHopIp())
2593                 .sorted().collect(toList());
2594         String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
2595         long groupId = nextHopManager.createNextHopPointer(lbGroupKey);
2596         int index = 0;
2597         List<ActionInfo> actionInfos = new ArrayList<>();
2598         for (AdjacencyResult adjResult : adjacencyResults) {
2599             String nextHopIp = adjResult.getNextHopIp();
2600             java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
2601             if (!optionalLabel.isPresent()) {
2602                 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
2603                 continue;
2604             }
2605             long label = optionalLabel.get();
2606
2607             actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
2608                     31, label));
2609         }
2610         List<InstructionInfo> instructions = new ArrayList<>();
2611         actionInfos.add(new ActionGroup(index, groupId));
2612         instructions.add(new InstructionApplyActions(actionInfos));
2613         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
2614     }
2615 }