vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17
  18 import java.math.BigInteger;
  19 import java.net.Inet4Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.List;
  27 import java.util.concurrent.BlockingQueue;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.function.Consumer;
  31
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  37 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
  38 import org.opendaylight.genius.mdsalutil.ActionInfo;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.MatchInfo;
  43 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  62 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.ServiceIndex;
  72 import org.opendaylight.genius.utils.batching.ActionableResource;
  73 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
  74 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
  75 import org.opendaylight.genius.utils.batching.ResourceHandler;
  76 import org.opendaylight.genius.utils.batching.SubTransaction;
  77 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
  78 import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
  79 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  80 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  81 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  82 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  83 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  84 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  85 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.TableKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
 138 import org.opendaylight.yangtools.yang.binding.DataObject;
 139 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 140 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 141 import org.slf4j.Logger;
 142 import org.slf4j.LoggerFactory;
 143
 144 public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
 145     implements AutoCloseable, ResourceHandler {
 146
 147     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
 148     private static final String FLOWID_PREFIX = "L3.";
 149     private static final int BATCH_SIZE = 1000;
 150     private static final int PERIODICITY = 500;
 151     private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
 152     private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
 153     private static final int LFIB_INTERVPN_PRIORITY = 15;
 154     public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 155
 156     private final DataBroker dataBroker;
 157     private final IMdsalApiManager mdsalManager;
 158     private final NexthopManager nextHopManager;
 159     private final OdlInterfaceRpcService interfaceManager;
 160     private final IdManagerService idManager;
 161     private final IVpnLinkService ivpnLinkService;
 162     List<SubTransaction> transactionObjects;
 163
 164     private static Integer batchSize;
 165     private static Integer batchInterval;
 166
 167     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
 168     private final ResourceBatchingManager resourceBatchingManager;
 169
 170     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
 171                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
 172                             final IdManagerService idManager, final IVpnLinkService ivpnLinkService) {
 173         super(VrfEntry.class, VrfEntryListener.class);
 174         this.dataBroker = dataBroker;
 175         this.mdsalManager = mdsalApiManager;
 176         this.nextHopManager = nexthopManager;
 177         this.interfaceManager = interfaceManager;
 178         this.idManager = idManager;
 179         this.ivpnLinkService = ivpnLinkService;
 180
 181         batchSize = Integer.getInteger("batch.size", BATCH_SIZE);
 182         batchInterval = Integer.getInteger("batch.wait.time", PERIODICITY);
 183         resourceBatchingManager = ResourceBatchingManager.getInstance();
 184         resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
 185         transactionObjects = new ArrayList<>();
 186     }
 187
 188     public void start() {
 189         LOG.info("{} start", getClass().getSimpleName());
 190         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 191     }
 192
 193     @Override
 194     protected VrfEntryListener getDataTreeChangeListener() {
 195         return VrfEntryListener.this;
 196     }
 197
 198     @Override
 199     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
 200         return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
 201     }
 202
 203     @Override
 204     public DataBroker getResourceBroker() {
 205         return dataBroker;
 206     }
 207
 208     public NexthopManager getNextHopManager() {
 209         return this.nextHopManager;
 210     }
 211
 212     @Override
 213     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
 214         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 215         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 216         LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
 217                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 218         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
 219             LOG.info("EVPN flows need to be programmed.");
 220             EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier,
 221                     vrfEntry, dataBroker, this);
 222             evpnVrfEntryProcessor.installFlows();
 223         } else {
 224             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 225                 createFibEntries(identifier, vrfEntry);
 226             } else {
 227                 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 228                 actResource.setAction(ActionableResource.CREATE);
 229                 actResource.setInstanceIdentifier(identifier);
 230                 actResource.setInstance(vrfEntry);
 231                 vrfEntryBufferQ.add(actResource);
 232             }
 233         }
 234         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
 235         LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
 236             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 237     }
 238
 239     @Override
 240     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
 241         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
 242         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 243         LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
 244                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 245         if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Vxlan)) {
 246             LOG.info("EVPN flows to be deleted");
 247             EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier, vrfEntry,
 248                     dataBroker, this);
 249             evpnVrfEntryProcessor.removeFlows();
 250         } else {
 251             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
 252                 deleteFibEntries(identifier, vrfEntry);
 253             } else {
 254                 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
 255                 actResource.setAction(ActionableResource.DELETE);
 256                 actResource.setInstanceIdentifier(identifier);
 257                 actResource.setInstance(vrfEntry);
 258                 vrfEntryBufferQ.add(actResource);
 259             }
 260         }
 261         leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
 262         LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
 263             rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
 264     }
 265
 266     @Override
 267     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
 268         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
 269
 270         final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
 271         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
 272         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {}",
 273             rd, update.getDestPrefix(), update.getRoutePaths());
 274         // Handle BGP Routes first
 275         if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
 276             ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
 277             actResource.setAction(ActionableResource.UPDATE);
 278             actResource.setInstanceIdentifier(identifier);
 279             actResource.setInstance(update);
 280             actResource.setOldInstance(original);
 281             vrfEntryBufferQ.add(actResource);
 282             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 283                 rd, update.getDestPrefix(), update.getRoutePaths());
 284             return;
 285         }
 286
 287         // Handle Vpn Interface driven Routes next (ie., STATIC and LOCAL)
 288         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(update.getOrigin()))) {
 289             List<RoutePaths> originalRoutePath = original.getRoutePaths();
 290             List<RoutePaths> updateRoutePath = update.getRoutePaths();
 291             LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
 292
 293             // If original VRF Entry had nexthop null , but update VRF Entry
 294             // has nexthop , route needs to be created on remote Dpns
 295             if (((originalRoutePath == null) || (originalRoutePath.isEmpty())
 296                 && (updateRoutePath != null) && (!updateRoutePath.isEmpty()))) {
 297                 // TODO(vivek): Though ugly, Not handling this code now, as each
 298                 // tep add event will invoke flow addition
 299                 LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
 300                     update.getDestPrefix());
 301                 return;
 302             }
 303
 304             // If original VRF Entry had valid nexthop , but update VRF Entry
 305             // has nexthop empty'ed out, route needs to be removed from remote Dpns
 306             if (((updateRoutePath == null) || (updateRoutePath.isEmpty())
 307                 && (originalRoutePath != null) && (!originalRoutePath.isEmpty()))) {
 308                 LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
 309                     update.getDestPrefix());
 310                 return;
 311             }
 312             createFibEntries(identifier, update);
 313             LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 314                 rd, update.getDestPrefix(), update.getRoutePaths());
 315             return;
 316         }
 317
 318         /* Handl all other route origins */
 319         createFibEntries(identifier, update);
 320
 321         LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
 322             rd, update.getDestPrefix(), update.getRoutePaths());
 323     }
 324
 325     @Override
 326     public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 327                        Object original, Object update, List<SubTransaction> transactionObjects) {
 328         this.transactionObjects = transactionObjects;
 329         if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
 330             createFibEntries(tx, identifier, (VrfEntry) update);
 331         }
 332     }
 333
 334     @Override
 335     public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 336                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 337         this.transactionObjects = transactionObjects;
 338         if (vrfEntry instanceof VrfEntry) {
 339             createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 340         }
 341     }
 342
 343     @Override
 344     public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
 345                        Object vrfEntry, List<SubTransaction> transactionObjects) {
 346         this.transactionObjects = transactionObjects;
 347         if (vrfEntry instanceof VrfEntry) {
 348             deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
 349         }
 350     }
 351
 352     @Override
 353     public int getBatchSize() {
 354         return batchSize;
 355     }
 356
 357     @Override
 358     public int getBatchInterval() {
 359         return batchInterval;
 360     }
 361
 362     @Override
 363     public LogicalDatastoreType getDatastoreType() {
 364         return LogicalDatastoreType.CONFIGURATION;
 365     }
 366
 367     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
 368         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 369
 370         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 371         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 372         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 373                 + " has null vpnId!");
 374         final Collection<VpnToDpnList> vpnToDpnList;
 375         if (vrfEntry.getParentVpnRd() != null
 376                 && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
 377             VpnInstanceOpDataEntry parentVpnInstance = getVpnInstance(vrfEntry.getParentVpnRd());
 378             vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
 379                 vpnInstance.getVpnToDpnList();
 380         } else {
 381             vpnToDpnList = vpnInstance.getVpnToDpnList();
 382         }
 383
 384         final Long vpnId = vpnInstance.getVpnId();
 385         final String rd = vrfTableKey.getRouteDistinguisher();
 386         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
 387         if (subnetRoute != null) {
 388             final long elanTag = subnetRoute.getElantag();
 389             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
 390                     rd, vrfEntry.getDestPrefix(), elanTag);
 391             if (vpnToDpnList != null) {
 392                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 393                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
 394                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 395                     for (final VpnToDpnList curDpn : vpnToDpnList) {
 396                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 397                             installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
 398                         }
 399                     }
 400                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 401                     futures.add(tx.submit());
 402                     return futures;
 403                 });
 404             }
 405             return;
 406         }
 407         // ping responder for router interfaces
 408         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
 409             return;
 410         }
 411
 412         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 413         if (!localDpnIdList.isEmpty()) {
 414             if (vpnToDpnList != null) {
 415                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 416                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
 417                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 418                     for (VpnToDpnList vpnDpn : vpnToDpnList) {
 419                         if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
 420                             if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 421                                 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry,
 422                                         tx);
 423                             }
 424                         }
 425                     }
 426                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 427                     futures.add(tx.submit());
 428                     return futures;
 429                 });
 430             }
 431         }
 432
 433         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
 434         if (optVpnUuid.isPresent()) {
 435             Optional<InterVpnLinkDataComposite> optInterVpnLink =
 436                 InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
 437             LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
 438             if (optInterVpnLink.isPresent()) {
 439                 InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
 440                 String vpnUuid = optVpnUuid.get();
 441                 FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
 442                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
 443                         // This is an static route that points to the other endpoint of an InterVpnLink
 444                         // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
 445                         installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
 446                         installInterVpnRouteInLFib(rd, vrfEntry);
 447                     }
 448                 });
 449             }
 450         }
 451     }
 452
 453     /*
 454       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
 455       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
 456       provided by ResourceBatchingManager
 457      */
 458     private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
 459                                   final VrfEntry vrfEntry) {
 460         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 461
 462         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
 463         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
 464         Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
 465             + " has null vpnId!");
 466
 467         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
 468         if (vpnToDpnList != null) {
 469             for (VpnToDpnList vpnDpn : vpnToDpnList) {
 470                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
 471                     createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
 472                 }
 473             }
 474         }
 475     }
 476
 477     // FIXME: Refactoring needed here.
 478     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
 479     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
 480                                    int addOrRemove) {
 481         Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 482         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 483
 484         String prefix = vrfEntry.getDestPrefix();
 485         List<String> nextHopsList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 486         // Label is used only for logging in subsequent method calls.
 487         //TODO : This label is not needed here. Can be removed. Hence using a default value.
 488         Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(0L);
 489         String rd = vrfTableKey.getRouteDistinguisher();
 490         LOG.trace("leakRouteIfNeeded: srcVpnRd={}  prefix={}  nhList={}  label={}", rd, prefix, nextHopsList, label);
 491
 492         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 493         if (vpnInstance == null) {
 494             LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
 495                       prefix, label, nextHopsList, rd);
 496             return;
 497         }
 498         String vpnUuid = vpnInstance.getVpnInstanceName();
 499         if (vpnUuid == null || vpnUuid.isEmpty()) {
 500             LOG.warn("Could not find suitable VPN UUID for rd={}.  vrfEntry=[prefix={}  nhList={}  label={}]",
 501                      rd, prefix, nextHopsList, label);
 502             return;
 503         }
 504
 505         ivpnLinkService.leakRouteIfNeeded(vpnUuid, prefix, nextHopsList, label.intValue(),
 506                                           RouteOrigin.value(vrfEntry.getOrigin()), addOrRemove);
 507     }
 508
 509     private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
 510         LOG.debug("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 511         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
 512         prefixBuilder.setDpnId(lri.getDpnId());
 513         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
 514         prefixBuilder.setIpAddress(lri.getPrefix());
 515         // Increment the refCount here
 516         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 517             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 518         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
 519         if (!isPresentInList) {
 520             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 521             List<String> vpnInstanceNames = lri.getVpnInstanceList();
 522             vpnInstanceNames.add(vpnInstanceName);
 523             builder.setVpnInstanceList(vpnInstanceNames);
 524             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 525                 FibUtil.DEFAULT_CALLBACK);
 526         } else {
 527             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
 528         }
 529         return prefixBuilder.build();
 530     }
 531
 532     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
 533                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
 534         Boolean wrTxPresent = true;
 535         if (tx == null) {
 536             wrTxPresent = false;
 537             tx = dataBroker.newWriteOnlyTransaction();
 538         }
 539         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
 540             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 541             synchronized (label.toString().intern()) {
 542                 LabelRouteInfo lri = getLabelRouteInfo(label);
 543                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 544
 545                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 546                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 547                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
 548                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
 549                             String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 550                             if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
 551                                 updateVpnReferencesInLri(lri, vpnInstanceName, false);
 552                             }
 553                         }
 554                     }
 555                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 556                             label, lri.getVpnInterfaceName(), lri.getDpnId());
 557                 }
 558             }
 559         });
 560         final List<InstructionInfo> instructions = new ArrayList<>();
 561         BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(24))
 562             .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
 563         instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 564         instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 565         makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 566
 567         if (vrfEntry.getRoutePaths() != null) {
 568             for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
 569                 if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 570                     List<ActionInfo> actionsInfos = new ArrayList<>();
 571                     // reinitialize instructions list for LFIB Table
 572                     final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
 573
 574                     actionsInfos.add(new ActionPopMpls());
 575                     LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
 576                     LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
 577                             MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
 578                     LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 579
 580                     makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
 581                             NwConstants.ADD_FLOW, tx);
 582                 }
 583             }
 584         }
 585         if (!wrTxPresent) {
 586             tx.submit();
 587         }
 588     }
 589
 590     /*
 591      * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
 592      * LportDispatcher table (via table 80)
 593      */
 594     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
 595         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
 596         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
 597         // packet is commuted from Vpn2 to Vpn1.
 598         Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
 599         if (!vpnNameOpc.isPresent()) {
 600             LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
 601             return;
 602         }
 603
 604         String vpnName = vpnNameOpc.get();
 605         List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
 606         boolean interVpnLinkFound = false;
 607         for (InterVpnLink interVpnLink : interVpnLinks) {
 608             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
 609             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
 610                 && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
 611             if (vpnIs1stEndpoint || vpnIs2ndEndpoint) {
 612                 interVpnLinkFound = true;
 613
 614                 Optional<InterVpnLinkState> vpnLinkState =
 615                     FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
 616                 if (!vpnLinkState.isPresent()
 617                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active)) {
 618                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
 619                         interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
 620                         interVpnLink.getSecondEndpoint().getVpnUuid().getValue());
 621                     return;
 622                 }
 623
 624                 List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
 625                     : vpnLinkState.get().getSecondEndpointState().getDpId();
 626                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
 627                     : vpnLinkState.get().getFirstEndpointState().getLportTag();
 628
 629                 LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
 630
 631                 for (BigInteger dpId : targetDpns) {
 632                     List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 633
 634                     List<InstructionInfo> instructions =
 635                         Arrays.asList(new InstructionApplyActions(actionsInfos),
 636                             new InstructionWriteMetadata(
 637                                 MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
 638                                     ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
 639                                         NwConstants.L3VPN_SERVICE_INDEX)),
 640                                 MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
 641                             new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
 642
 643                     FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(interVpnRoutePathLabel -> {
 644                         List<String> interVpnNextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 645                         LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for "
 646                                 + "InterVpnLink {} in LFIB",
 647                                 vrfEntry.getDestPrefix(), interVpnRoutePathLabel, interVpnNextHopList,
 648                                 dpId, interVpnLink.getName());
 649
 650                         makeLFibTableEntry(dpId, interVpnRoutePathLabel, instructions, LFIB_INTERVPN_PRIORITY,
 651                                 NwConstants.ADD_FLOW, null);
 652                     });
 653                 }
 654
 655                 break;
 656             }
 657         }
 658
 659         if (!interVpnLinkFound) {
 660             LOG.warn("VrfEntry=[prefix={} route-paths={}] for VPN {} has origin INTERVPN but "
 661                 + "no InterVpnLink could be found",
 662                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), rd);
 663         }
 664     }
 665
 666
 667     /*
 668      * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
 669      */
 670     private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
 671                                                final VrfEntry vrfEntry, long vpnTag) {
 672         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
 673         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
 674             && vrfEntry.getRoutePaths().size() == 1);
 675         String destination = vrfEntry.getDestPrefix();
 676         String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
 677         String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 678
 679         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
 680         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
 681         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
 682         if (interVpnLink.getState().or(State.Error) != State.Active) {
 683             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
 684                 destination, nextHop, interVpnLinkName);
 685             return;
 686         }
 687
 688         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
 689         if (!optOtherEndpointLportTag.isPresent()) {
 690             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
 691                 vpnUuid, interVpnLinkName);
 692             return;
 693         }
 694
 695         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
 696         if (targetDpns.isEmpty()) {
 697             LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
 698                 vpnUuid, interVpnLinkName);
 699             return;
 700         }
 701
 702         String[] values = destination.split("/");
 703         String destPrefixIpAddress = values[0];
 704         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 705
 706         List<MatchInfo> matches = new ArrayList<>();
 707         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
 708         matches.add(MatchEthernetType.IPV4);
 709
 710         if (prefixLength != 0) {
 711             matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
 712         }
 713
 714         List<Instruction> instructions =
 715             Arrays.asList(new InstructionWriteMetadata(
 716                     MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
 717                         ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
 718                             .L3VPN_SERVICE_INDEX)),
 719                     MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
 720                 new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 721
 722         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
 723         String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
 724         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
 725             COOKIE_VM_FIB_TABLE, matches, instructions);
 726
 727         LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
 728             vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 729
 730         for (BigInteger dpId : targetDpns) {
 731
 732             LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
 733                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
 734                 dpId, interVpnLink.getInterVpnLinkName());
 735
 736             mdsalManager.installFlow(dpId, flowEntity);
 737         }
 738     }
 739
 740
 741     // TODO Clean up the exception handling
 742     @SuppressWarnings("checkstyle:IllegalCatch")
 743     private <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 744                                                     InstanceIdentifier<T> path) {
 745
 746         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 747
 748         Optional<T> result = Optional.absent();
 749         try {
 750             result = tx.read(datastoreType, path).get();
 751         } catch (Exception e) {
 752             throw new RuntimeException(e);
 753         }
 754
 755         return result;
 756     }
 757
 758     private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
 759         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 760         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 761
 762         if (localNextHopInfo == null) {
 763             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 764             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
 765                     FibUtil.getVpnNameFromId(dataBroker, vpnId), rd, vrfEntry.getDestPrefix());
 766             if (extraRouteOptional.isPresent()) {
 767                 Routes extraRoute = extraRouteOptional.get();
 768                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
 769                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 770                     if (nextHopIp != null) {
 771                         localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp
 772                                 + NwConstants.IPV4PREFIX);
 773                         if (localNextHopInfo != null) {
 774                             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 775                         }
 776                     }
 777                 }
 778             }
 779         } else {
 780             returnLocalDpnId.add(localNextHopInfo.getDpnId());
 781         }
 782
 783         return returnLocalDpnId;
 784     }
 785
 786     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
 787         List<BigInteger> returnLocalDpnId = new ArrayList<>();
 788         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
 789         String localNextHopIP = vrfEntry.getDestPrefix();
 790         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 791         if (localNextHopInfo == null) {
 792             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
 793             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
 794                     vpnName, usedRds, localNextHopIP);
 795             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
 796             vpnExtraRoutes.stream().forEach(extraRoute -> {
 797                 Prefixes localNextHopInfoLocal = FibUtil.getPrefixToInterface(dataBroker,
 798                         vpnId, extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
 799                 BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfoLocal, vrfEntry.getDestPrefix(),
 800                         vpnId, rd, vrfEntry, vpnId, extraRoute, vpnExtraRoutes);
 801                 returnLocalDpnId.add(dpnId);
 802             });
 803             if (localNextHopInfo == null) {
 804             /* imported routes case */
 805                 if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
 806                     java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 807                     if (optionalLabel.isPresent()) {
 808                         Long label = optionalLabel.get();
 809                         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 810                         synchronized (label.toString().intern()) {
 811                             LabelRouteInfo lri = getLabelRouteInfo(label);
 812                             if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
 813                                 Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
 814                                         FibUtil.getVpnInstanceOpData(dataBroker, rd);
 815                                 if (vpnInstanceOpDataEntryOptional.isPresent()) {
 816                                     String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
 817                                     if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
 818                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
 819                                         localNextHopIP = lri.getPrefix();
 820                                     } else {
 821                                         localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
 822                                         localNextHopIP = lri.getPrefix();
 823                                     }
 824                                 }
 825                                 if (localNextHopInfo != null) {
 826                                     LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
 827                                             label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
 828                                     if (vpnExtraRoutes == null || vpnExtraRoutes.isEmpty()) {
 829                                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
 830                                                 vpnId, rd, vrfEntry, lri.getParentVpnid(), null, vpnExtraRoutes);
 831                                         returnLocalDpnId.add(dpnId);
 832                                     } else {
 833                                         for (Routes extraRoutes : vpnExtraRoutes) {
 834                                             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo,
 835                                                     localNextHopIP,
 836                                                     vpnId, rd, vrfEntry, lri.getParentVpnid(),
 837                                                     extraRoutes, vpnExtraRoutes);
 838                                             returnLocalDpnId.add(dpnId);
 839                                         }
 840                                     }
 841                                 }
 842                             }
 843                         }
 844                     }
 845                 }
 846             }
 847             if (returnLocalDpnId.isEmpty()) {
 848                 LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
 849             }
 850         } else {
 851             BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
 852                     rd, vrfEntry, vpnId, /*routes*/ null, /*vpnExtraRoutes*/ null);
 853             returnLocalDpnId.add(dpnId);
 854         }
 855         return returnLocalDpnId;
 856     }
 857
 858     private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
 859                                                 final Long vpnId, final String rd,
 860                                                 final VrfEntry vrfEntry, Long parentVpnId,
 861                                                 Routes routes, List<Routes> vpnExtraRoutes) {
 862         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 863         if (localNextHopInfo != null) {
 864             long groupId;
 865             long localGroupId;
 866             final BigInteger dpnId = localNextHopInfo.getDpnId();
 867             if (Boolean.TRUE.equals(localNextHopInfo.isNatPrefix())) {
 868                 LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip local dpn {} FIB processing",
 869                         vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
 870                 return dpnId;
 871             }
 872             String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
 873             if (routes != null) {
 874                 groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
 875                         vpnExtraRoutes);
 876                 localGroupId = nextHopManager.getLocalNextHopGroup(parentVpnId,
 877                         routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
 878             } else {
 879                 groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId,
 880                         localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix(),
 881                         vrfEntry.getGatewayMacAddress(), jobKey);
 882                 localGroupId = groupId;
 883             }
 884             if (groupId == FibConstants.INVALID_GROUP_ID) {
 885                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
 886                     vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
 887                 return BigInteger.ZERO;
 888             }
 889             final List<InstructionInfo> instructions = Collections.singletonList(
 890                 new InstructionApplyActions(
 891                     Collections.singletonList(new ActionGroup(groupId))));
 892             final List<InstructionInfo> lfibinstructions = Collections.singletonList(
 893                 new InstructionApplyActions(
 894                     Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
 895             java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
 896             List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
 897             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 898                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
 899                     dpnId, localNextHopInfo.getVpnInterfaceName(), optLabel);
 900             } else {
 901                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
 902                     + "LFib and Terminating table entries will not be created.",
 903                     rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
 904             }
 905             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
 906
 907             dataStoreCoordinator.enqueueJob(jobKey,
 908                 () -> {
 909                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
 910                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
 911                     optLabel.ifPresent(label -> {
 912                         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
 913                             makeLFibTableEntry(dpnId, label, lfibinstructions,
 914                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
 915                             makeTunnelTableEntry(dpnId, label, localGroupId, tx);
 916                         }
 917                     });
 918
 919                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 920                     futures.add(tx.submit());
 921                     return futures;
 922                 });
 923             return dpnId;
 924         }
 925         LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}",
 926                 vrfEntry.getDestPrefix(), rd, vpnName);
 927         return BigInteger.ZERO;
 928     }
 929
 930     private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
 931         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
 932         return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id).isPresent();
 933     }
 934
 935     private LabelRouteInfo getLabelRouteInfo(Long label) {
 936         InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
 937             .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
 938         Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
 939         if (opResult.isPresent()) {
 940             return opResult.get();
 941         }
 942         return null;
 943     }
 944
 945     private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
 946         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
 947         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
 948             .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
 949         if (lri == null) {
 950             return true;
 951         }
 952         List<String> vpnInstancesList = lri.getVpnInstanceList() != null
 953             ? lri.getVpnInstanceList() : new ArrayList<>();
 954         if (vpnInstancesList.contains(vpnInstanceName)) {
 955             LOG.debug("vpninstance {} name is present", vpnInstanceName);
 956             vpnInstancesList.remove(vpnInstanceName);
 957         }
 958         if (vpnInstancesList.size() == 0) {
 959             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
 960             if (tx != null) {
 961                 tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
 962             } else {
 963                 FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
 964             }
 965             return true;
 966         } else {
 967             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
 968             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
 969             FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(),
 970                 FibUtil.DEFAULT_CALLBACK);
 971         }
 972         return false;
 973     }
 974
 975     void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
 976                                       WriteTransaction tx) {
 977         List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 978
 979         createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 980
 981         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
 982             dpId, label, groupId);
 983     }
 984
 985     public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
 986                                                 WriteTransaction tx) {
 987         List<MatchInfo> mkMatches = new ArrayList<>();
 988
 989         LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
 990             destDpId, label, actionsInfos);
 991
 992         // Matching metadata
 993         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
 994         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 995
 996         List<InstructionInfo> mkInstructions = new ArrayList<>();
 997         mkInstructions.add(new InstructionApplyActions(actionsInfos));
 998
 999         FlowEntity terminatingServiceTableFlowEntity =
1000             MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1001             getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
1002                 String.format("%s:%d", "TST Flow Entry ", label),
1003             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
1004
1005         FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
1006
1007         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
1008
1009         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
1010         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1011             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1012             .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
1013             .child(Flow.class, flowKey).build();
1014         tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
1015     }
1016
1017     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
1018         FlowEntity flowEntity;
1019         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
1020         List<MatchInfo> mkMatches = new ArrayList<>();
1021         // Matching metadata
1022         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
1023         flowEntity = MDSALUtil.buildFlowEntity(dpId,
1024             NwConstants.INTERNAL_TUNNEL_TABLE,
1025             getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
1026             5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
1027             COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
1028         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
1029         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
1030         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1031             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1032             .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
1033
1034         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1035         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
1036     }
1037
1038     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
1039         List<BigInteger> returnLocalDpnId = new ArrayList<>();
1040         Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1041         String localNextHopIP = vrfEntry.getDestPrefix();
1042         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1043
1044         if (localNextHopInfo == null) {
1045             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1046             if (usedRds.size() > 1) {
1047                 LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
1048                         vrfEntry.getDestPrefix(), vpnName, rd);
1049                 return returnLocalDpnId;
1050             }
1051             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
1052             //in the vpn
1053             Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
1054                     vpnName, rd, vrfEntry.getDestPrefix());
1055             if (extraRouteOptional.isPresent()) {
1056                 Routes extraRoute = extraRouteOptional.get();
1057                 localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId,
1058                         extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
1059                 if (localNextHopInfo != null) {
1060                     BigInteger dpnId = localNextHopInfo.getDpnId();
1061                     if (!dpnId.equals(BigInteger.ZERO)) {
1062                         nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
1063                                 vrfEntry.getDestPrefix(), /*listBucketInfo*/ null, /*remove*/ false);
1064                         returnLocalDpnId.add(dpnId);
1065                     }
1066                 }
1067             }
1068
1069             if (localNextHopInfo == null) {
1070                 /* Imported VRF entry */
1071                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1072                 if (optionalLabel.isPresent()) {
1073                     Long label = optionalLabel.get();
1074                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1075                     LabelRouteInfo lri = getLabelRouteInfo(label);
1076                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1077                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1078                         prefixBuilder.setDpnId(lri.getDpnId());
1079                         BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), localNextHopIP,
1080                                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1081                         if (!dpnId.equals(BigInteger.ZERO)) {
1082                             returnLocalDpnId.add(dpnId);
1083                         }
1084                     }
1085                 }
1086             }
1087
1088         } else {
1089             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
1090                 vpnId, rd, vrfEntry, false /*isExtraRoute*/);
1091             if (!dpnId.equals(BigInteger.ZERO)) {
1092                 returnLocalDpnId.add(dpnId);
1093             }
1094         }
1095
1096         return returnLocalDpnId;
1097     }
1098
1099     private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
1100                                                 final Long vpnId, final String rd,
1101                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
1102         if (localNextHopInfo != null) {
1103             final BigInteger dpnId = localNextHopInfo.getDpnId();
1104             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1105             dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-"
1106                     + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
1107                 () -> {
1108                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1109                     makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
1110                         NwConstants.DEL_FLOW, tx);
1111                     if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1112                         FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1113                             makeLFibTableEntry(dpnId, label, null /* instructions */,
1114                                     DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1115                             removeTunnelTableEntry(dpnId, label, tx);
1116                         });
1117                     }
1118                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1119                     futures.add(tx.submit());
1120                     return futures;
1121                 });
1122             //TODO: verify below adjacency call need to be optimized (?)
1123             deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
1124             return dpnId;
1125         }
1126         return BigInteger.ZERO;
1127     }
1128
1129     static  InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
1130             String ipPrefix) {
1131         return InstanceIdentifier.builder(VpnToExtraroutes.class)
1132                 .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
1133                         new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
1134     }
1135
1136     public Routes getVpnToExtraroute(String vpnRd, String destPrefix) {
1137         Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, vpnRd);
1138         if (optVpnName.isPresent()) {
1139             InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
1140                     optVpnName.get(), vpnRd, destPrefix);
1141             return FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
1142         }
1143         return null;
1144     }
1145
1146     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
1147             final VrfEntry vrfEntry, WriteTransaction tx) {
1148         Boolean wrTxPresent = true;
1149         if (tx == null) {
1150             wrTxPresent = false;
1151             tx = dataBroker.newWriteOnlyTransaction();
1152         }
1153         String rd = vrfTableKey.getRouteDistinguisher();
1154         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
1155         LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
1156                 vrfEntry.getDestPrefix(), rd, remoteDpnId);
1157
1158         List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1159         if (adjacencyResults == null || adjacencyResults.isEmpty()) {
1160             LOG.error("Could not get interface for route-paths: {} in vpn {}",
1161                     vrfEntry.getRoutePaths(), rd);
1162             LOG.warn("Failed to add Route: {} in vpn: {}",
1163                     vrfEntry.getDestPrefix(), rd);
1164             return;
1165         }
1166         if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
1167             programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1168         } else {
1169             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
1170             List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
1171                     vpnName, usedRds, vrfEntry.getDestPrefix());
1172             if (!vpnExtraRoutes.isEmpty()) {
1173                 List<InstructionInfo> instructions = new ArrayList<>();
1174                 long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
1175                         null, vpnExtraRoutes);
1176                 if (groupId == FibConstants.INVALID_GROUP_ID) {
1177                     LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
1178                             vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
1179                     return;
1180                 }
1181                 List<ActionInfo> actionInfos =
1182                         Collections.singletonList(new ActionGroup(groupId));
1183                 instructions.add(new InstructionApplyActions(actionInfos));
1184                 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1185             } else {
1186                 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1187             }
1188         }
1189         if (!wrTxPresent) {
1190             tx.submit();
1191         }
1192         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
1193     }
1194
1195     private void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
1196             final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
1197         List<InstructionInfo> instructions = new ArrayList<>();
1198         for (AdjacencyResult adjacencyResult : adjacencyResults) {
1199             List<ActionInfo> actionInfos = new ArrayList<>();
1200             String egressInterface = adjacencyResult.getInterfaceName();
1201             if (FibUtil.isTunnelInterface(adjacencyResult)) {
1202                 addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos);
1203             } else {
1204                 addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1205             }
1206             List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
1207                     actionInfos.size());
1208             if (egressActions.isEmpty()) {
1209                 LOG.error(
1210                         "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
1211                                 + "Aborting remote FIB entry creation.",
1212                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1213                 return;
1214             }
1215             actionInfos.addAll(egressActions);
1216             instructions.add(new InstructionApplyActions(actionInfos));
1217         }
1218         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
1219     }
1220
1221     private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
1222         if (vrfEntry.getMac() != null) {
1223             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1224                 new MacAddress(vrfEntry.getMac())));
1225             return;
1226         }
1227
1228         String ipPrefix = vrfEntry.getDestPrefix();
1229         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
1230         if (prefixInfo == null) {
1231             LOG.debug("No prefix info found for prefix {}", ipPrefix);
1232             return;
1233         }
1234
1235         String ifName = prefixInfo.getVpnInterfaceName();
1236         if (ifName == null) {
1237             LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
1238             return;
1239         }
1240
1241         String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
1242         if (macAddress == null) {
1243             LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
1244             return;
1245         }
1246
1247         actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
1248     }
1249
1250     private void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, long vpnId, VrfEntry vrfEntry,
1251                                            List<ActionInfo> actionInfos) {
1252         Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(interfaceManager,
1253                 adjacencyResult.getInterfaceName());
1254         // TODO - For now have added routePath into adjacencyResult so that we know for which
1255         // routePath this result is built for. If this is not possible construct a map which does
1256         // the same.
1257         String nextHopIp = adjacencyResult.getNextHopIp();
1258         java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
1259         if (!optionalLabel.isPresent()) {
1260             LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
1261             return;
1262         }
1263         long label = optionalLabel.get();
1264         if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
1265             LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
1266             actionInfos.add(new ActionPushMpls());
1267             actionInfos.add(new ActionSetFieldMplsLabel(label));
1268             actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
1269         } else {
1270             BigInteger tunnelId;
1271             // FIXME vxlan vni bit set is not working properly with OVS.need to
1272             // revisit
1273             if (tunnelType.equals(TunnelTypeVxlan.class)) {
1274                 tunnelId = BigInteger.valueOf(label);
1275             } else {
1276                 tunnelId = BigInteger.valueOf(label);
1277             }
1278
1279             LOG.debug("adding set tunnel id action for label {}", label);
1280             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1281             addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
1282         }
1283     }
1284
1285     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
1286         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
1287         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1288         if (dpnInVpn.isPresent()) {
1289             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
1290             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
1291
1292             if (vpnInterfaces.remove(currVpnInterface)) {
1293                 if (vpnInterfaces.isEmpty()) {
1294                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
1295                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1296                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
1297                 } else {
1298                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
1299                     FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
1300                         VpnInterfaces.class,
1301                         new VpnInterfacesKey(intfName)));
1302                 }
1303             }
1304         }
1305     }
1306
1307     void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
1308     /* Get interface info from prefix to interface mapping;
1309         Use the interface info to get the corresponding vpn interface op DS entry,
1310         remove the adjacency corresponding to this fib entry.
1311         If adjacency removed is the last adjacency, clean up the following:
1312          - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
1313          - prefix to interface entry
1314          - vpn interface op DS
1315      */
1316         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
1317         Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
1318         Routes extraRoute = null;
1319         if (prefixInfo == null) {
1320             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
1321             if (extraRoute != null) {
1322                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
1323                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
1324                     if (nextHopIp != null) {
1325                         prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp
1326                                 + NwConstants.IPV4PREFIX);
1327                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1328                     }
1329                 }
1330             }
1331             if (prefixInfo == null) {
1332                 java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1333                 if (optionalLabel.isPresent()) {
1334                     Long label = optionalLabel.get();
1335                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1336                     LabelRouteInfo lri = getLabelRouteInfo(label);
1337                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1338                         PrefixesBuilder prefixBuilder = new PrefixesBuilder();
1339                         prefixBuilder.setDpnId(lri.getDpnId());
1340                         prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
1341                         prefixBuilder.setIpAddress(lri.getPrefix());
1342                         prefixInfo = prefixBuilder.build();
1343                         LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
1344                                 label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
1345                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1346                     }
1347                 }
1348             }
1349         } else {
1350             checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
1351         }
1352     }
1353
1354     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
1355                                           final VrfEntry vrfEntry, final Routes extraRoute) {
1356
1357         if (prefixInfo == null) {
1358             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
1359             return; //Don't have any info for this prefix (shouldn't happen); need to return
1360         }
1361
1362         if (Boolean.TRUE.equals(prefixInfo.isNatPrefix())) {
1363             LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
1364                     vrfEntry.getDestPrefix(), vpnId, rd);
1365             return;
1366         }
1367
1368         String ifName = prefixInfo.getVpnInterfaceName();
1369         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1370         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
1371             new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
1372     }
1373
1374     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
1375         Prefixes prefixInfo;
1376         Long vpnId;
1377         String rd;
1378         VrfEntry vrfEntry;
1379         Routes extraRoute;
1380
1381         CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
1382                                          final VrfEntry vrfEntry, final Routes extraRoute) {
1383             this.prefixInfo = prefixInfo;
1384             this.vpnId = vpnId;
1385             this.rd = rd;
1386             this.vrfEntry = vrfEntry;
1387             this.extraRoute = extraRoute;
1388         }
1389
1390         @Override
1391         public List<ListenableFuture<Void>> call() throws Exception {
1392             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
1393             // to call the respective helpers.
1394             WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1395
1396             //First Cleanup LabelRouteInfo
1397             //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
1398             if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Mplsgre)) {
1399                 FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
1400                     List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1401                     synchronized (label.toString().intern()) {
1402                         LabelRouteInfo lri = getLabelRouteInfo(label);
1403                         if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
1404                                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
1405                             Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1406                                     FibUtil.getVpnInstanceOpData(dataBroker, rd);
1407                             String vpnInstanceName = "";
1408                             if (vpnInstanceOpDataEntryOptional.isPresent()) {
1409                                 vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1410                             }
1411                             boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, writeOperTxn);
1412                             if (lriRemoved) {
1413                                 String parentRd = lri.getParentVpnRd();
1414                                 FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1415                                         FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1416                             }
1417                         } else {
1418                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1419                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1420                         }
1421                     }
1422                 });
1423             }
1424             String ifName = prefixInfo.getVpnInterfaceName();
1425             Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1426                 FibUtil.getVpnInterfaceIdentifier(ifName));
1427             if (optvpnInterface.isPresent()) {
1428                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
1429                 if (vpnId != associatedVpnId) {
1430                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
1431                         vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
1432                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
1433                     return null;
1434                 } else {
1435                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
1436                         vrfEntry.getDestPrefix(), associatedVpnId);
1437                 }
1438             }
1439             if (extraRoute != null) {
1440                 Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, rd);
1441                 if (optVpnName.isPresent()) {
1442                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1443                             getVpnToExtrarouteIdentifier(optVpnName.get(), rd, vrfEntry.getDestPrefix()));
1444                 }
1445             }
1446             Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
1447                 FibUtil.getAdjListPath(ifName));
1448             int numAdj = 0;
1449             if (optAdjacencies.isPresent()) {
1450                 numAdj = optAdjacencies.get().getAdjacency().size();
1451             }
1452             //remove adjacency corr to prefix
1453             if (numAdj > 1) {
1454                 LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId,
1455                         vrfEntry.getDestPrefix());
1456                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
1457                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
1458             } else {
1459                 //this is last adjacency (or) no more adjacency left for this vpn interface, so
1460                 //clean up the vpn interface from DpnToVpn list
1461                 LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
1462                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName));
1463             }
1464             List<ListenableFuture<Void>> futures = new ArrayList<>();
1465             futures.add(writeOperTxn.submit());
1466             return futures;
1467         }
1468     }
1469
1470     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
1471         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1472         final String rd = vrfTableKey.getRouteDistinguisher();
1473         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1474         if (vpnInstance == null) {
1475             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1476             return;
1477         }
1478         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1479         long elanTag = 0L;
1480         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1481         final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1482         List<String> nextHopAddressList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1483         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1484         if (subnetRoute != null) {
1485             elanTag = subnetRoute.getElantag();
1486             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
1487                 rd, vrfEntry.getDestPrefix(), elanTag);
1488             if (vpnToDpnList != null) {
1489                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1490                 dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
1491                     () -> {
1492                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1493
1494                         for (final VpnToDpnList curDpn : vpnToDpnList) {
1495
1496                             makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
1497                                 vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
1498                             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
1499                                 optionalLabel.ifPresent(label -> {
1500                                     makeLFibTableEntry(curDpn.getDpnId(), label, null,
1501                                             DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
1502                                 });
1503                             }
1504                         }
1505                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1506                         futures.add(tx.submit());
1507                         return futures;
1508                     });
1509             }
1510             optionalLabel.ifPresent(label -> {
1511                 synchronized (label.toString().intern()) {
1512                     LabelRouteInfo lri = getLabelRouteInfo(label);
1513                     if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
1514                         Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
1515                                 FibUtil.getVpnInstanceOpData(dataBroker, rd);
1516                         String vpnInstanceName = "";
1517                         if (vpnInstanceOpDataEntryOptional.isPresent()) {
1518                             vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
1519                         }
1520                         boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
1521                         if (lriRemoved) {
1522                             String parentRd = lri.getParentVpnRd();
1523                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1524                                     FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
1525                             LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
1526                                     + "labelRouteInfo cleared", label, rd,
1527                                     vrfEntry.getDestPrefix());
1528                         }
1529                     } else {
1530                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
1531                                 FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
1532                         LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
1533                                 label, rd, vrfEntry.getDestPrefix());
1534                     }
1535                 }
1536             });
1537             return;
1538         }
1539         if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
1540             return;
1541         }
1542
1543         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
1544             vrfTableKey.getRouteDistinguisher(), vrfEntry);
1545         if (vpnToDpnList != null) {
1546             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1547                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1548             String usedRd = null;
1549             Optional<Routes> extraRouteOptional;
1550             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1551             if (usedRds != null && !usedRds.isEmpty()) {
1552                 if (usedRds.size() > 1) {
1553                     LOG.error("The extra route prefix is still present in some DPNs");
1554                     return ;
1555                 } else {
1556                     // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
1557                     //is not present in any other DPN
1558                     extraRouteOptional = VpnExtraRouteHelper
1559                             .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
1560                 }
1561             } else {
1562                 extraRouteOptional = Optional.absent();
1563             }
1564             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1565             dataStoreCoordinator.enqueueJob("FIB-" + usedRd + "-" + vrfEntry.getDestPrefix(),
1566                 () -> {
1567                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1568
1569                     if (localDpnIdList.size() <= 0) {
1570                         for (VpnToDpnList curDpn : vpnToDpnList) {
1571                             if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1572                                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1573                                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1574                                         vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1575                                 }
1576                             } else {
1577                                 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
1578                                     vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1579                             }
1580                         }
1581                     } else {
1582                         for (BigInteger localDpnId : localDpnIdList) {
1583                             for (VpnToDpnList curDpn : vpnToDpnList) {
1584                                 if (!curDpn.getDpnId().equals(localDpnId)) {
1585                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1586                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1587                                             deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1588                                                 vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1589                                         }
1590                                     } else {
1591                                         deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
1592                                             vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
1593                                     }
1594                                 }
1595                             }
1596                         }
1597                     }
1598                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1599                     futures.add(tx.submit());
1600                     return futures;
1601                 });
1602         }
1603
1604         //The flow/group entry has been deleted from config DS; need to clean up associated operational
1605         //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
1606         cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
1607
1608         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
1609         // of the interVpnLink.
1610         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
1611         if (optVpnUuid.isPresent()) {
1612             String vpnUuid = optVpnUuid.get();
1613             FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
1614                 Optional<InterVpnLinkDataComposite> optInterVpnLink =
1615                         InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
1616                 if (optInterVpnLink.isPresent()) {
1617                     InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
1618                     if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
1619                         // This is route that points to the other endpoint of an InterVpnLink
1620                         // In that case, we should look for the FIB table pointing to
1621                         // LPortDispatcher table and remove it.
1622                         removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
1623                                 interVpnLink.isFirstEndpointVpnName(rd),
1624                                 vrfEntry);
1625                     }
1626                 }
1627             });
1628         }
1629
1630     }
1631
1632     /*
1633       Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
1634       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
1635       provided by ResourceBatchingManager
1636      */
1637     private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
1638                                   final VrfEntry vrfEntry) {
1639         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
1640         String rd = vrfTableKey.getRouteDistinguisher();
1641         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
1642         if (vpnInstance == null) {
1643             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
1644             return;
1645         }
1646         String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
1647         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
1648         if (vpnToDpnList != null) {
1649             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
1650                     vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
1651             Optional<Routes> extraRouteOptional;
1652             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
1653             if (usedRds != null && !usedRds.isEmpty()) {
1654                 if (usedRds.size() > 1) {
1655                     LOG.error("The extra route prefix is still present in some DPNs");
1656                     return ;
1657                 } else {
1658                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
1659                             usedRds.get(0), vrfEntry.getDestPrefix());
1660                 }
1661             } else {
1662                 extraRouteOptional = Optional.absent();
1663             }
1664             for (VpnToDpnList curDpn : vpnToDpnList) {
1665                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
1666                     deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey,
1667                             vrfEntry, extraRouteOptional, writeTx);
1668                 }
1669             }
1670         }
1671     }
1672
1673     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
1674             final long vpnId, final VrfTablesKey vrfTableKey,
1675             final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional, WriteTransaction tx) {
1676
1677         Boolean wrTxPresent = true;
1678         if (tx == null) {
1679             wrTxPresent = false;
1680             tx = dataBroker.newWriteOnlyTransaction();
1681         }
1682
1683         LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
1684                 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
1685         String rd = vrfTableKey.getRouteDistinguisher();
1686
1687         if (localDpnId != null && localDpnId != BigInteger.ZERO) {
1688             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
1689             if (extraRouteOptional.isPresent()) {
1690                 nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
1691             }
1692             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1693             return;
1694         }
1695
1696         // below two reads are kept as is, until best way is found to identify dpnID
1697         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
1698         if (extraRouteOptional.isPresent()) {
1699             nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
1700         } else {
1701             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
1702         }
1703         if (!wrTxPresent) {
1704             tx.submit();
1705         }
1706     }
1707
1708     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
1709                                            VrfEntry vrfEntry, String rd, WriteTransaction tx) {
1710         boolean isRemoteRoute = true;
1711         if (localNextHopInfo != null) {
1712             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
1713         }
1714         if (isRemoteRoute) {
1715             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
1716             return true;
1717         } else {
1718             LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
1719                 rd, vrfEntry.getDestPrefix(), remoteDpnId);
1720             return false;
1721         }
1722     }
1723
1724     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
1725             String rd, WriteTransaction tx) {
1726         // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
1727         // the adjacencyResults.
1728         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1729             List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
1730             if (!adjacencyResults.isEmpty()) {
1731                 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
1732                 return;
1733             }
1734         }
1735         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
1736         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
1737     }
1738
1739     private long get(byte[] rawIpAddress) {
1740         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
1741             + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
1742     }
1743
1744     void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
1745                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
1746         Boolean wrTxPresent = true;
1747         if (tx == null) {
1748             wrTxPresent = false;
1749             tx = dataBroker.newWriteOnlyTransaction();
1750         }
1751
1752         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
1753         String[] values = vrfEntry.getDestPrefix().split("/");
1754         String ipAddress = values[0];
1755         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
1756         if (addOrRemove == NwConstants.ADD_FLOW) {
1757             LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
1758         } else {
1759             LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
1760         }
1761         InetAddress destPrefix;
1762         try {
1763             destPrefix = InetAddress.getByName(ipAddress);
1764         } catch (UnknownHostException e) {
1765             LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
1766             return;
1767         }
1768
1769         List<MatchInfo> matches = new ArrayList<>();
1770
1771         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
1772
1773         if (destPrefix instanceof Inet4Address) {
1774             matches.add(MatchEthernetType.IPV4);
1775             if (prefixLength != 0) {
1776                 matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
1777             }
1778         } else {
1779             matches.add(MatchEthernetType.IPV6);
1780             if (prefixLength != 0) {
1781                 matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
1782             }
1783         }
1784
1785         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
1786         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
1787         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
1788             flowRef, 0, 0,
1789             COOKIE_VM_FIB_TABLE, matches, instructions);
1790         Flow flow = flowEntity.getFlowBuilder().build();
1791         String flowId = flowEntity.getFlowId();
1792         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1793         Node nodeDpn = buildDpnNode(dpId);
1794
1795         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1796             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1797             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1798
1799         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
1800             SubTransaction subTransaction = new SubTransactionImpl();
1801             if (addOrRemove == NwConstants.ADD_FLOW) {
1802                 subTransaction.setInstanceIdentifier(flowInstanceId);
1803                 subTransaction.setInstance(flow);
1804                 subTransaction.setAction(SubTransaction.CREATE);
1805             } else {
1806                 subTransaction.setInstanceIdentifier(flowInstanceId);
1807                 subTransaction.setAction(SubTransaction.DELETE);
1808             }
1809             transactionObjects.add(subTransaction);
1810         }
1811
1812         if (addOrRemove == NwConstants.ADD_FLOW) {
1813             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1814         } else {
1815             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1816         }
1817
1818         if (!wrTxPresent) {
1819             tx.submit();
1820         }
1821     }
1822
1823     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
1824     Node buildDpnNode(BigInteger dpnId) {
1825         NodeId nodeId = new NodeId("openflow:" + dpnId);
1826         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
1827
1828         return nodeDpn;
1829     }
1830
1831     private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions, int priority,
1832                                     int addOrRemove, WriteTransaction tx) {
1833         Boolean wrTxPresent = true;
1834         if (tx == null) {
1835             wrTxPresent = false;
1836             tx = dataBroker.newWriteOnlyTransaction();
1837         }
1838
1839         List<MatchInfo> matches = new ArrayList<>();
1840         matches.add(MatchEthernetType.MPLS_UNICAST);
1841         matches.add(new MatchMplsLabel(label));
1842
1843         // Install the flow entry in L3_LFIB_TABLE
1844         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
1845
1846         FlowEntity flowEntity;
1847         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
1848             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
1849         Flow flow = flowEntity.getFlowBuilder().build();
1850         String flowId = flowEntity.getFlowId();
1851         FlowKey flowKey = new FlowKey(new FlowId(flowId));
1852         Node nodeDpn = buildDpnNode(dpId);
1853         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
1854             .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
1855             .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
1856
1857         if (addOrRemove == NwConstants.ADD_FLOW) {
1858             tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
1859         } else {
1860             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
1861         }
1862         if (!wrTxPresent) {
1863             tx.submit();
1864         }
1865
1866         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
1867             dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
1868     }
1869
1870     void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
1871                                       final String ipPrefixAddress) {
1872         LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
1873         try {
1874             nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
1875         } catch (NullPointerException e) {
1876             LOG.trace("", e);
1877         }
1878     }
1879
1880     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
1881                                     final FutureCallback<List<Void>> callback) {
1882         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
1883         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1884         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1885         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1886         if (!vrfTable.isPresent()) {
1887             LOG.warn("VRF Table not yet available for RD {}", rd);
1888             if (callback != null) {
1889                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1890                 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1891                 Futures.addCallback(listenableFuture, callback);
1892             }
1893             return;
1894         }
1895         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1896         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1897             () -> {
1898                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1899                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
1900                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
1901                     for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
1902                         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
1903                         if (subnetRoute != null) {
1904                             long elanTag = subnetRoute.getElantag();
1905                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
1906                             continue;
1907                         }
1908                         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
1909                         if (routerInt != null) {
1910                             LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
1911                                 rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
1912                             installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
1913                                 routerInt.getIpAddress(),
1914                                 new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
1915                             continue;
1916                         }
1917                         //Handle local flow creation for imports
1918                         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
1919                             java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
1920                             if (optionalLabel.isPresent()) {
1921                                 List<String> nextHopList = FibUtil.getNextHopListFromRoutePaths(vrfEntry);
1922                                 LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
1923                                 if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
1924                                     if (lri.getDpnId().equals(dpnId)) {
1925                                         createLocalFibEntry(vpnId, rd, vrfEntry);
1926                                         continue;
1927                                     }
1928                                 }
1929                             }
1930                         }
1931
1932                         boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
1933                                 vrfEntry, dpnId);
1934                         if (shouldCreateRemoteFibEntry) {
1935                             LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}",
1936                                     vrfEntry, dpnId);
1937                             createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
1938                         }
1939                     }
1940                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
1941                     futures.add(tx.submit());
1942                 }
1943                 if (callback != null) {
1944                     ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
1945                     Futures.addCallback(listenableFuture, callback);
1946                 }
1947                 return futures;
1948             });
1949     }
1950
1951     public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1952                                             final String localNextHopIp, final String remoteNextHopIp) {
1953         LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1954             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1955         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1956         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1957         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1958         if (vrfTable.isPresent()) {
1959             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1960             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1961                 () -> {
1962                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1963                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1964                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1965                         vrfTable.get().getVrfEntry().stream()
1966                             .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
1967                             .forEach(
1968                                 getConsumerForCreatingRemoteFib(dpnId, vpnId,
1969                                         rd, remoteNextHopIp, vrfTable,
1970                                         writeCfgTxn));
1971                         futures.add(writeCfgTxn.submit());
1972                     }
1973                     return futures;
1974                 });
1975         }
1976     }
1977
1978     public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
1979                                             final String localNextHopIp, final String remoteNextHopIp) {
1980         LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
1981             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
1982         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
1983         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
1984         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1985         if (vrfTable.isPresent()) {
1986             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
1987             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
1988                 () -> {
1989                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1990                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
1991                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
1992                         // Handle Vpn Interface driven Routes only (i.e., STATIC and LOCAL)
1993                         vrfTable.get().getVrfEntry().stream()
1994                             .filter(vrfEntry -> {
1995                                 /* Ignore SubnetRoute entry */
1996                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1997                                         vrfEntry.getOrigin())));
1998                             })
1999                             .forEach(getConsumerForCreatingRemoteFib(dpnId, vpnId,
2000                                        rd, remoteNextHopIp, vrfTable,
2001                                        writeCfgTxn));
2002                         futures.add(writeCfgTxn.submit());
2003                     }
2004                     return futures;
2005                 });
2006         }
2007     }
2008
2009     public void manageRemoteRouteOnDPN(final boolean action,
2010                                        final BigInteger localDpnId,
2011                                        final long vpnId,
2012                                        final String rd,
2013                                        final String destPrefix,
2014                                        final String destTepIp,
2015                                        final long label) {
2016         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2017
2018         if (vpnInstance == null) {
2019             LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
2020             return;
2021         }
2022         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2023         dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
2024             () -> {
2025                 List<ListenableFuture<Void>> futures = new ArrayList<>();
2026                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
2027                     WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
2028                     VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
2029                     VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
2030                     if (vrfEntry == null) {
2031                         return futures;
2032                     }
2033                     LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
2034                         action, localDpnId, vpnId, rd, destPrefix);
2035                     List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
2036                     VrfEntry modVrfEntry;
2037                     if (routePathList == null || (routePathList.isEmpty())) {
2038                         modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
2039                                 Collections.singletonList(destTepIp),
2040                                 RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
2041                     } else {
2042                         modVrfEntry = vrfEntry;
2043                     }
2044
2045                     if (action == true) {
2046                         LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
2047                         createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
2048                     } else {
2049                         LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
2050                         List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
2051                                 vrfEntry.getDestPrefix());
2052                         if (usedRds.size() > 1) {
2053                             LOG.debug("The extra route prefix is still present in some DPNs");
2054                             return futures;
2055                         }
2056                         //Is this fib route an extra route? If yes, get the nexthop which would be
2057                         //an adjacency in the vpn
2058                         Optional<Routes> extraRouteOptional = Optional.absent();
2059                         if (usedRds.size() != 0) {
2060                             extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
2061                                     FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId()),
2062                                     usedRds.get(0), vrfEntry.getDestPrefix());
2063                         }
2064                         deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
2065                                 extraRouteOptional, writeTransaction);
2066                     }
2067                     futures.add(writeTransaction.submit());
2068                 }
2069                 return futures;
2070             });
2071     }
2072
2073     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
2074                                  final FutureCallback<List<Void>> callback) {
2075         LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
2076         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2077         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2078         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2079         if (vrfTable.isPresent()) {
2080             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2081             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2082                 () -> {
2083                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2084                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2085                         WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2086                         for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
2087                                 /* Handle subnet routes here */
2088                             SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
2089                             if (subnetRoute != null) {
2090                                 LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
2091                                         vrfEntry.getDestPrefix(),
2092                                         dpnId, rd);
2093                                 makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
2094                                 java.util.Optional.ofNullable(vrfEntry.getRoutePaths()).ifPresent(routePaths -> {
2095                                     routePaths.stream().forEach(routePath -> {
2096                                         makeLFibTableEntry(dpnId, routePath.getLabel(), null,
2097                                                 DEFAULT_FIB_FLOW_PRIORITY,
2098                                                 NwConstants.DEL_FLOW, tx);
2099                                         LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} "
2100                                                 + "for rd {} prefix {}",
2101                                                 routePath.getLabel(), rd,
2102                                                 vrfEntry.getDestPrefix());
2103                                     });
2104                                 });
2105                                 continue;
2106                             }
2107                             // ping responder for router interfaces
2108                             RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2109                             if (routerInt != null) {
2110                                 LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
2111                                     rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
2112                                 installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
2113                                     routerInt.getIpAddress(),
2114                                     new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
2115                                 continue;
2116                             }
2117                             // Passing null as we don't know the dpn
2118                             // to which prefix is attached at this point
2119                             List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
2120                                     vrfEntry.getDestPrefix());
2121                             String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
2122                             Optional<Routes> extraRouteOptional;
2123                             //Is this fib route an extra route? If yes, get the nexthop which would be
2124                             //an adjacency in the vpn
2125                             if (usedRds != null && !usedRds.isEmpty()) {
2126                                 if (usedRds.size() > 1) {
2127                                     LOG.error("The extra route prefix is still present in some DPNs");
2128                                     return futures;
2129                                 } else {
2130                                     extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
2131                                             usedRds.get(0), vrfEntry.getDestPrefix());
2132
2133                                 }
2134                             } else {
2135                                 extraRouteOptional = Optional.absent();
2136                             }
2137                             deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
2138                                     extraRouteOptional, tx);
2139                         }
2140                         futures.add(tx.submit());
2141                         if (callback != null) {
2142                             ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
2143                             Futures.addCallback(listenableFuture, callback);
2144                         }
2145                     }
2146                     return futures;
2147                 });
2148         }
2149     }
2150
2151     public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2152                                            final String localNextHopIp, final String remoteNextHopIp) {
2153         LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2154                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2155             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2156         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2157         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2158         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2159         if (vrfTable.isPresent()) {
2160             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2161             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2162                 () -> {
2163                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2164                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2165                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2166                         vrfTable.get().getVrfEntry().stream()
2167                             .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
2168                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
2169                                 remoteNextHopIp, vrfTable, writeCfgTxn));
2170                         futures.add(writeCfgTxn.submit());
2171                     }
2172                     return futures;
2173                 });
2174         }
2175     }
2176
2177     public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
2178                                            final String localNextHopIp, final String remoteNextHopIp) {
2179         LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
2180                 + " localNexthopIp {} , remoteNexhtHopIp {}",
2181             dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
2182         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
2183         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
2184         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2185         if (vrfTable.isPresent()) {
2186             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
2187             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
2188                 () -> {
2189                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2190                     synchronized (vpnInstance.getVpnInstanceName().intern()) {
2191                         WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
2192                         vrfTable.get().getVrfEntry().stream()
2193                             .filter(vrfEntry -> {
2194                                 /* Ignore SubnetRoute entry */
2195                                 return (FibHelper.isControllerManagedVpnInterfaceRoute(
2196                                         RouteOrigin.value(vrfEntry.getOrigin())));
2197                             })
2198                             .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId,
2199                                 rd, remoteNextHopIp, vrfTable, writeCfgTxn));
2200                         futures.add(writeCfgTxn.submit());
2201                     }
2202                     return futures;
2203                 });
2204         }
2205     }
2206
2207     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
2208         InstanceIdentifierBuilder<VrfTables> idBuilder =
2209             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
2210         InstanceIdentifier<VrfTables> id = idBuilder.build();
2211         return id;
2212     }
2213
2214     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
2215         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + label
2216                 + NwConstants.FLOWID_SEPARATOR + priority;
2217     }
2218
2219     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
2220         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + rd
2221                 + NwConstants.FLOWID_SEPARATOR + priority + NwConstants.FLOWID_SEPARATOR + destPrefix.getHostAddress();
2222     }
2223
2224     private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
2225         return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
2226                 .FLOWID_SEPARATOR + nextHop;
2227     }
2228
2229     protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
2230                                                      final VrfEntry vrfEntry, String rd) {
2231         List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2232         FibHelper.sortIpAddress(routePaths);
2233         List<AdjacencyResult> adjacencyList = new ArrayList<>();
2234         List<String> prefixIpList = new ArrayList<>();
2235         LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
2236             remoteDpnId, vpnId, vrfEntry);
2237         try {
2238             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
2239                 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
2240                 List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
2241                         FibUtil.getVpnNameFromId(dataBroker, vpnId), usedRds, vrfEntry.getDestPrefix());
2242                 if (vpnExtraRoutes.isEmpty()) {
2243                     prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2244                 } else {
2245                     List<String> prefixIpListLocal = new ArrayList<>();
2246                     vpnExtraRoutes.stream().forEach(route -> {
2247                         route.getNexthopIpList().stream().forEach(extraRouteIp -> {
2248                             prefixIpListLocal.add(extraRouteIp + NwConstants.IPV4PREFIX);
2249                         });
2250                     });
2251                     prefixIpList = prefixIpListLocal;
2252                 }
2253             } else {
2254                 prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
2255             }
2256
2257             for (String prefixIp : prefixIpList) {
2258                 if (routePaths == null || routePaths.isEmpty()) {
2259                     LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
2260                     AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2261                           prefixIp, null);
2262                     addAdjacencyResultToList(adjacencyList, adjacencyResult);
2263                     continue;
2264                 }
2265                 adjacencyList.addAll(routePaths.stream()
2266                         .map(routePath -> {
2267                             LOG.debug("NextHop IP for destination {} is {}", prefixIp,
2268                                     routePath.getNexthopAddress());
2269                             return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
2270                                     prefixIp, routePath.getNexthopAddress());
2271                         })
2272                         .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
2273                         .distinct()
2274                         .collect(toList()));
2275             }
2276         } catch (NullPointerException e) {
2277             LOG.trace("", e);
2278         }
2279         return adjacencyList;
2280     }
2281
2282     private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
2283         if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
2284             adjacencyList.add(adjacencyResult);
2285         }
2286     }
2287
2288     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
2289         InstanceIdentifier<VpnInstanceOpDataEntry> id =
2290             InstanceIdentifier.create(VpnInstanceOpData.class)
2291                 .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
2292         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
2293             FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
2294         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
2295     }
2296
2297     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
2298         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
2299                 + tableMiss + FLOWID_PREFIX;
2300     }
2301
2302     /*
2303      * Install flow entry in protocol table to forward mpls
2304      * coming through gre tunnel to LFIB table.
2305      */
2306     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
2307         final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
2308         // Instruction to goto L3 InterfaceTable
2309         List<InstructionInfo> instructions = new ArrayList<>();
2310         instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
2311         List<MatchInfo> matches = new ArrayList<>();
2312         matches.add(MatchEthernetType.MPLS_UNICAST);
2313         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2314             getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
2315                 NwConstants.L3_LFIB_TABLE),
2316             DEFAULT_FIB_FLOW_PRIORITY,
2317             "Protocol Table For LFIB",
2318             0, 0,
2319             cookieProtocolTable,
2320             matches, instructions);
2321
2322         if (addOrRemove == NwConstants.ADD_FLOW) {
2323             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
2324             mdsalManager.installFlow(flowEntityToLfib);
2325         } else {
2326             mdsalManager.removeFlow(flowEntityToLfib);
2327         }
2328     }
2329
2330     public List<String> printFibEntries() {
2331         List<String> result = new ArrayList<>();
2332         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
2333         result.add("-------------------------------------------------------------------");
2334         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
2335         Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
2336         if (fibEntries.isPresent()) {
2337             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
2338             for (VrfTables vrfTable : vrfTables) {
2339                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
2340                     List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
2341                     for (RoutePaths routePath : routePaths) {
2342                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
2343                             vrfTable.getRouteDistinguisher(),
2344                             vrfEntry.getDestPrefix(), routePath.getNexthopAddress(),
2345                             routePath.getLabel(), vrfEntry.getOrigin()));
2346                     }
2347                     if (routePaths.isEmpty()) {
2348                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s",
2349                             vrfTable.getRouteDistinguisher(),
2350                             vrfEntry.getDestPrefix(), "local", vrfEntry.getOrigin()));
2351                     }
2352                 }
2353             }
2354         }
2355         return result;
2356     }
2357
2358
2359     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
2360         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2361             .child(VrfTables.class, new VrfTablesKey(rd))
2362             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2363         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
2364         if (vrfEntry.isPresent()) {
2365             return vrfEntry.get();
2366         }
2367         return null;
2368     }
2369
2370     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
2371         InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
2372             .child(VrfTables.class, new VrfTablesKey(rd))
2373             .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
2374         return vrfEntryId;
2375     }
2376
2377     protected Boolean isIpv4Address(String ipAddress) {
2378         try {
2379             InetAddress address = InetAddress.getByName(ipAddress);
2380             if (address instanceof Inet4Address) {
2381                 return true;
2382             }
2383         } catch (UnknownHostException e) {
2384             LOG.warn("Invalid ip address {}", ipAddress, e);
2385             return false;
2386         }
2387         return false;
2388     }
2389
2390     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
2391                                               long vpnId, int addOrRemove) {
2392         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
2393         if (routerInt == null) {
2394             return false;
2395         }
2396         if (vpnToDpnList != null) {
2397             String routerId = routerInt.getUuid();
2398             String macAddress = routerInt.getMacAddress();
2399             String ipValue = routerInt.getIpAddress();
2400             LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
2401                     routerId, ipValue, macAddress);
2402             for (VpnToDpnList vpnDpn : vpnToDpnList) {
2403                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
2404                     installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
2405                             new MacAddress(macAddress), addOrRemove);
2406                 }
2407             }
2408         }
2409         return true;
2410     }
2411
2412     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
2413                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
2414         String[] subSplit = routerInternalIp.split("/");
2415         if (!isIpv4Address(subSplit[0])) {
2416             // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
2417             return;
2418         }
2419
2420         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2421         if (!optionalLabel.isPresent()) {
2422             LOG.warn("Routes paths not present. Exiting installRouterFibEntry");
2423             return;
2424         }
2425         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
2426         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
2427             dpnId, routerInternalIp, vpnId, subSplit[0]);
2428
2429         List<MatchInfo> matches = new ArrayList<>();
2430
2431         matches.add(MatchIpProtocol.ICMP);
2432         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
2433         matches.add(new MatchIcmpv4((short) 8, (short) 0));
2434         matches.add(MatchEthernetType.IPV4);
2435         matches.add(new MatchIpv4Destination(subSplit[0], "32"));
2436
2437         List<ActionInfo> actionsInfos = new ArrayList<>();
2438
2439         // Set Eth Src and Eth Dst
2440         actionsInfos.add(new ActionMoveSourceDestinationEth());
2441         actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
2442
2443         // Move Ip Src to Ip Dst
2444         actionsInfos.add(new ActionMoveSourceDestinationIp());
2445         actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
2446
2447         // Set the ICMP type to 0 (echo reply)
2448         actionsInfos.add(new ActionSetIcmpType((short) 0));
2449
2450         actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
2451
2452         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2453
2454         List<InstructionInfo> instructions = new ArrayList<>();
2455
2456         instructions.add(new InstructionApplyActions(actionsInfos));
2457
2458         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
2459         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, optionalLabel.get(),
2460                 priority);
2461
2462         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
2463             0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
2464
2465         if (addOrRemove == NwConstants.ADD_FLOW) {
2466             mdsalManager.installFlow(flowEntity);
2467         } else {
2468             mdsalManager.removeFlow(flowEntity);
2469         }
2470     }
2471
2472     public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
2473                                              final boolean isVpnFirstEndPoint,
2474                                              final VrfEntry vrfEntry) {
2475         Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
2476             && vrfEntry.getRoutePaths().size() == 1);
2477         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
2478
2479         if (!interVpnLinkState.isPresent()) {
2480             LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
2481             return;
2482         }
2483
2484         List<BigInteger> targetDpns =
2485             isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
2486                 : interVpnLinkState.get().getSecondEndpointState().getDpId();
2487
2488         java.util.Optional<String> optionalNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
2489         java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
2490
2491         // delete from FIB
2492         //
2493         optionalNextHop.ifPresent(nextHop -> {
2494             String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
2495             FlowKey flowKey = new FlowKey(new FlowId(flowRef));
2496             Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
2497                     .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
2498
2499             LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
2500                     interVpnLinkName, flowRef);
2501
2502             for (BigInteger dpId : targetDpns) {
2503                 LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
2504                         vrfEntry.getDestPrefix(), nextHop,
2505                         dpId, interVpnLinkName);
2506
2507                 mdsalManager.removeFlow(dpId, flow);
2508             }
2509         });
2510
2511         // delete from LFIB
2512         //
2513         optionalLabel.ifPresent(label -> {
2514             LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
2515
2516             WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
2517             for (BigInteger dpId : targetDpns) {
2518                 LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
2519                         vrfEntry.getDestPrefix(), label,
2520                         dpId, interVpnLinkName);
2521                 makeLFibTableEntry(dpId, label, null /* no instructions */,
2522                         LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
2523             }
2524             tx.submit();
2525         });
2526     }
2527
2528     private Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
2529             final BigInteger dpnId, final long vpnId, final String rd,
2530             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2531             WriteTransaction writeCfgTxn) {
2532         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2533                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2534                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2535                 .findFirst()
2536                 .ifPresent(routes -> {
2537                     LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
2538                             vrfEntry.getDestPrefix(), rd, dpnId);
2539                     createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
2540                 });
2541     }
2542
2543     private Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
2544             final BigInteger dpnId, final long vpnId, final String rd,
2545             final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
2546             WriteTransaction writeCfgTxn) {
2547         return vrfEntry -> vrfEntry.getRoutePaths().stream()
2548                 .filter(routes -> !routes.getNexthopAddress().isEmpty()
2549                         && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
2550                 .findFirst()
2551                 .ifPresent(routes -> {
2552                     LOG.trace(" deleting remote FIB entry {}", vrfEntry);
2553                     deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
2554                             Optional.absent(), writeCfgTxn);
2555                 });
2556     }
2557
2558     private boolean isPrefixAndNextHopPresentInLri(String prefix,
2559             List<String> nextHopAddressList, LabelRouteInfo lri) {
2560         return lri != null && lri.getPrefix().equals(prefix)
2561                 && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
2562     }
2563
2564     private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
2565         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
2566             return true;
2567         }
2568
2569         Prefixes prefix = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
2570         if (prefix != null) {
2571             BigInteger prefixDpnId = prefix.getDpnId();
2572             if (prefixDpnId == dpnId) {
2573                 LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
2574                         vrfEntry, dpnId);
2575                 return false;
2576             }
2577         }
2578
2579         return true;
2580     }
2581
2582     private void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId, final long vpnId,
2583             final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
2584         Preconditions.checkArgument(vrfEntry.getRoutePaths().size() <= 2);
2585
2586         if (adjacencyResults.size() == 1) {
2587             programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
2588             return;
2589         }
2590         // ECMP Use case, point to LB group. Move the mpls label accordingly.
2591         List<String> tunnelList =
2592                 adjacencyResults.stream()
2593                 .map(adjacencyResult -> adjacencyResult.getNextHopIp())
2594                 .sorted().collect(toList());
2595         String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
2596         long groupId = nextHopManager.createNextHopPointer(lbGroupKey);
2597         int index = 0;
2598         List<ActionInfo> actionInfos = new ArrayList<>();
2599         for (AdjacencyResult adjResult : adjacencyResults) {
2600             String nextHopIp = adjResult.getNextHopIp();
2601             java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
2602             if (!optionalLabel.isPresent()) {
2603                 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
2604                 continue;
2605             }
2606             long label = optionalLabel.get();
2607
2608             actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
2609                     31, label));
2610         }
2611         List<InstructionInfo> instructions = new ArrayList<>();
2612         actionInfos.add(new ActionGroup(index, groupId));
2613         instructions.add(new InstructionApplyActions(actionInfos));
2614         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
2615     }
2616 }