BUG-3874: BGP Manager / fib-show shell cmd triggers an exception on usage
[netvirt.git] / vpnservice / fibmanager / fibmanager-impl / src / main / java / org / opendaylight / netvirt / fibmanager / VrfEntryListener.java
old mode 100644 (file)
new mode 100755 (executable)
index 6ba4bc4..3bc2d1e
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
@@ -7,6 +7,8 @@
  */
 package org.opendaylight.netvirt.fibmanager;
 
+import static java.util.stream.Collectors.toList;
+
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.FutureCallback;
@@ -14,35 +16,60 @@ import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 
 import java.math.BigInteger;
+import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Consumer;
 
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import javax.inject.Singleton;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
-import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
-import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
+import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
 import org.opendaylight.genius.mdsalutil.ActionInfo;
-import org.opendaylight.genius.mdsalutil.ActionType;
 import org.opendaylight.genius.mdsalutil.FlowEntity;
 import org.opendaylight.genius.mdsalutil.InstructionInfo;
-import org.opendaylight.genius.mdsalutil.InstructionType;
 import org.opendaylight.genius.mdsalutil.MDSALUtil;
-import org.opendaylight.genius.mdsalutil.MatchFieldType;
 import org.opendaylight.genius.mdsalutil.MatchInfo;
 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
+import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
+import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
+import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
+import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
+import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
+import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
+import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
+import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
+import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
+import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
+import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
+import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
+import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
+import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
+import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
+import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
+import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
 import org.opendaylight.genius.utils.ServiceIndex;
 import org.opendaylight.genius.utils.batching.ActionableResource;
 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
@@ -50,11 +77,15 @@ import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
 import org.opendaylight.genius.utils.batching.ResourceHandler;
 import org.opendaylight.genius.utils.batching.SubTransaction;
 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
+import org.opendaylight.netvirt.elanmanager.api.IElanService;
+import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
+import org.opendaylight.netvirt.fibmanager.api.FibHelper;
 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
-import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
+import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table;
@@ -66,10 +97,8 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instru
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeInputBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetTunnelTypeOutput;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
@@ -77,6 +106,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.N
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
@@ -84,93 +114,91 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroutes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.Vpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.VpnKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.ExtraRoutesKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.RoutesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
-import org.opendaylight.yangtools.yang.binding.DataObject;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
-import org.opendaylight.yangtools.yang.common.RpcResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener> implements AutoCloseable, ResourceHandler {
+@Singleton
+public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry, VrfEntryListener>
+    implements AutoCloseable, ResourceHandler {
+
     private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class);
     private static final String FLOWID_PREFIX = "L3.";
+    private static final int BATCH_SIZE = 1000;
+    private static final int PERIODICITY = 500;
+    private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
+    private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
+    private static final int LFIB_INTERVPN_PRIORITY = 15;
+    public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
+
     private final DataBroker dataBroker;
     private final IMdsalApiManager mdsalManager;
-    private IVpnManager vpnmanager;
     private final NexthopManager nextHopManager;
-    private ItmRpcService itmManager;
     private final OdlInterfaceRpcService interfaceManager;
     private final IdManagerService idManager;
-    private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
-    private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
-    private static final int LFIB_INTERVPN_PRIORITY = 1;
-    private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
-    private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
-    public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
     List<SubTransaction> transactionObjects;
-    private static final int PERIODICITY = 500;
+
     private static Integer batchSize;
     private static Integer batchInterval;
-    private static final int BATCH_SIZE = 1000;
+
     private static BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
     private final ResourceBatchingManager resourceBatchingManager;
 
+    protected static boolean isOpenStackVniSemanticsEnforced;
+
+    @Inject
     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
                             final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager,
-                            final IdManagerService idManager) {
+                            final IdManagerService idManager,
+                            final IElanService elanManager) {
         super(VrfEntry.class, VrfEntryListener.class);
         this.dataBroker = dataBroker;
         this.mdsalManager = mdsalApiManager;
         this.nextHopManager = nexthopManager;
         this.interfaceManager = interfaceManager;
         this.idManager = idManager;
-        batchSize = Integer.getInteger("batch.size");
-        if (batchSize == null) {
-            batchSize = BATCH_SIZE;
-        }
-        batchInterval = Integer.getInteger("batch.wait.time");
-        if (batchInterval == null) {
-            batchInterval = PERIODICITY;
-        }
+
+        isOpenStackVniSemanticsEnforced = elanManager.isOpenStackVniSemanticsEnforced();
+        batchSize = Integer.getInteger("batch.size", BATCH_SIZE);
+        batchInterval = Integer.getInteger("batch.wait.time", PERIODICITY);
         resourceBatchingManager = ResourceBatchingManager.getInstance();
-        resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this);
+        resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
         transactionObjects = new ArrayList<>();
     }
 
-    public void start() {
-        LOG.info("{} start", getClass().getSimpleName());
+    @Override
+    @PostConstruct
+    public void init() {
+        LOG.info("{} init", getClass().getSimpleName());
         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
     }
 
     @Override
-    protected VrfEntryListener getDataTreeChangeListener() { return VrfEntryListener.this; }
+    protected VrfEntryListener getDataTreeChangeListener() {
+        return VrfEntryListener.this;
+    }
 
     @Override
     protected InstanceIdentifier<VrfEntry> getWildCardPath() {
@@ -182,88 +210,153 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         return dataBroker;
     }
 
+    public NexthopManager getNextHopManager() {
+        return this.nextHopManager;
+    }
+
     @Override
     protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
-        LOG.debug("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
-        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
-            createFibEntries(identifier, vrfEntry);
+        LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}",
+                rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
+        if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
+            LOG.info("EVPN flows need to be programmed.");
+            EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier,
+                    vrfEntry, dataBroker, this);
+            evpnVrfEntryProcessor.installFlows();
         } else {
-            ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
-            actResource.setAction(ActionableResource.CREATE);
-            actResource.setInstanceIdentifier(identifier);
-            actResource.setInstance(vrfEntry);
-            vrfEntryBufferQ.add(actResource);
-            leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
+            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
+                createFibEntries(identifier, vrfEntry);
+            } else {
+                ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
+                actResource.setAction(ActionableResource.CREATE);
+                actResource.setInstanceIdentifier(identifier);
+                actResource.setInstance(vrfEntry);
+                vrfEntryBufferQ.add(actResource);
+            }
         }
-        LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
+        LOG.info("ADD: Added Fib Entry rd {} prefix {} route-paths {}",
+                 rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
     }
 
     @Override
     protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
         Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
         String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
-        LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
-        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
-            deleteFibEntries(identifier, vrfEntry);
+        LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}",
+                rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
+        if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Vxlan)) {
+            LOG.info("EVPN flows to be deleted");
+            EVPNVrfEntryProcessor evpnVrfEntryProcessor = new EVPNVrfEntryProcessor(identifier, vrfEntry,
+                    dataBroker, this);
+            evpnVrfEntryProcessor.removeFlows();
         } else {
-            ActionableResource actResource = new ActionableResourceImpl(rd.toString() + vrfEntry.getDestPrefix());
-            actResource.setAction(ActionableResource.DELETE);
-            actResource.setInstanceIdentifier(identifier);
-            actResource.setInstance(vrfEntry);
-            vrfEntryBufferQ.add(actResource);
-            leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
+            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
+                deleteFibEntries(identifier, vrfEntry);
+            } else {
+                ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
+                actResource.setAction(ActionableResource.DELETE);
+                actResource.setInstanceIdentifier(identifier);
+                actResource.setInstance(vrfEntry);
+                vrfEntryBufferQ.add(actResource);
+            }
         }
-        LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
+        LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} route-paths {}",
+            rd, vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
     }
 
     @Override
     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
-        String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
-        LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
-                rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
-        if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
-            createFibEntries(identifier, update);
-        } else {
-            ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
+
+        final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
+        LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {}",
+            rd, update.getDestPrefix(), update.getRoutePaths());
+        // Handle BGP Routes first
+        if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
+            ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
             actResource.setAction(ActionableResource.UPDATE);
             actResource.setInstanceIdentifier(identifier);
             actResource.setInstance(update);
             actResource.setOldInstance(original);
             vrfEntryBufferQ.add(actResource);
+            LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
+                rd, update.getDestPrefix(), update.getRoutePaths());
+            return;
+        }
+
+        // Handle Vpn Interface driven Routes next (ie., STATIC and LOCAL)
+        if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(update.getOrigin()))) {
+            List<RoutePaths> originalRoutePath = original.getRoutePaths();
+            List<RoutePaths> updateRoutePath = update.getRoutePaths();
+            LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath);
+
+            // If original VRF Entry had nexthop null , but update VRF Entry
+            // has nexthop , route needs to be created on remote Dpns
+            if (((originalRoutePath == null) || (originalRoutePath.isEmpty())
+                && (updateRoutePath != null) && (!updateRoutePath.isEmpty()))) {
+                // TODO(vivek): Though ugly, Not handling this code now, as each
+                // tep add event will invoke flow addition
+                LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.",
+                    update.getDestPrefix());
+                return;
+            }
+
+            // If original VRF Entry had valid nexthop , but update VRF Entry
+            // has nexthop empty'ed out, route needs to be removed from remote Dpns
+            if (((updateRoutePath == null) || (updateRoutePath.isEmpty())
+                && (originalRoutePath != null) && (!originalRoutePath.isEmpty()))) {
+                LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.",
+                    update.getDestPrefix());
+                return;
+            }
+            //Update the used rds and vpntoextraroute containers only for the deleted nextHops.
+            List<String> nextHopsRemoved = FibHelper.getNextHopListFromRoutePaths(original);
+            nextHopsRemoved.removeAll(FibHelper.getNextHopListFromRoutePaths(update));
+            WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
+            nextHopsRemoved.parallelStream()
+                    .forEach(nextHopRemoved -> FibUtil.updateUsedRdAndVpnToExtraRoute(
+                             writeOperTxn, dataBroker, nextHopRemoved, rd,
+                             update.getDestPrefix()));
+            writeOperTxn.submit();
+            createFibEntries(identifier, update);
+            LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
+                rd, update.getDestPrefix(), update.getRoutePaths());
+            return;
         }
-        LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
-                rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
+
+        /* Handl all other route origins */
+        createFibEntries(identifier, update);
+
+        LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} route-paths {}",
+            rd, update.getDestPrefix(), update.getRoutePaths());
     }
 
     @Override
-    public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
+    public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
+                       Object original, Object update, List<SubTransaction> transactionObjects) {
         this.transactionObjects = transactionObjects;
-        if (vrfEntry instanceof VrfEntry) {
-            createFibEntries(tx, identifier, (VrfEntry)vrfEntry);
+        if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
+            createFibEntries(tx, identifier, (VrfEntry) update);
         }
     }
 
     @Override
-    public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List<SubTransaction> transactionObjects) {
+    public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
+                       Object vrfEntry, List<SubTransaction> transactionObjects) {
         this.transactionObjects = transactionObjects;
         if (vrfEntry instanceof VrfEntry) {
-            deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
+            createFibEntries(tx, identifier, (VrfEntry) vrfEntry);
         }
     }
 
     @Override
-    public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original,
-                       Object update, List<SubTransaction> transactionObjects) {
+    public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
+                       Object vrfEntry, List<SubTransaction> transactionObjects) {
         this.transactionObjects = transactionObjects;
-        if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) {
-            createFibEntries(tx, identifier, (VrfEntry)update);
+        if (vrfEntry instanceof VrfEntry) {
+            deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry);
         }
     }
 
@@ -284,12 +377,20 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
     private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
-
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
-        Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
+        Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
+                + " has null vpnId!");
+        final Collection<VpnToDpnList> vpnToDpnList;
+        if (vrfEntry.getParentVpnRd() != null
+                && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
+            VpnInstanceOpDataEntry parentVpnInstance = getVpnInstance(vrfEntry.getParentVpnRd());
+            vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() :
+                vpnInstance.getVpnToDpnList();
+        } else {
+            vpnToDpnList = vpnInstance.getVpnToDpnList();
+        }
 
-        final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
         final Long vpnId = vpnInstance.getVpnId();
         final String rd = vrfTableKey.getRouteDistinguisher();
         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
@@ -299,87 +400,79 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                     rd, vrfEntry.getDestPrefix(), elanTag);
             if (vpnToDpnList != null) {
                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
-                        new Callable<List<ListenableFuture<Void>>>() {
-                            @Override
-                            public List<ListenableFuture<Void>> call() throws Exception {
-                                WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-                                for (final VpnToDpnList curDpn : vpnToDpnList) {
-                                    if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
-                                        installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId.longValue(), vrfEntry, tx);
-                                    }
-                                }
-                                List<ListenableFuture<Void>> futures = new ArrayList<>();
-                                futures.add(tx.submit());
-                                return futures;
-                            }
-                        });
+                dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
+                    WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+                    for (final VpnToDpnList curDpn : vpnToDpnList) {
+                        if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
+                            installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
+                        }
+                    }
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    futures.add(tx.submit());
+                    return futures;
+                });
             }
             return;
         }
-
-        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
-            // When it is a leaked route, the LFIB and FIB goes a bit different.
-            installInterVpnRouteInLFib(rd, vrfEntry);
+        // ping responder for router interfaces
+        if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) {
             return;
         }
 
         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
-
-        if (vpnToDpnList != null) {
-            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-                            for (VpnToDpnList vpnDpn : vpnToDpnList) {
-                                if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
-                                    if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
-                                        createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                    }
-                                }
+        if (!localDpnIdList.isEmpty()) {
+            if (vpnToDpnList != null) {
+                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+                dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(), () -> {
+                    WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+                    for (VpnToDpnList vpnDpn : vpnToDpnList) {
+                        if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
+                            if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
+                                createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry,
+                                        tx);
                             }
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            futures.add(tx.submit());
-                            return futures;
                         }
-                    });
+                    }
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    futures.add(tx.submit());
+                    return futures;
+                });
+            }
         }
 
         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd);
-        if ( optVpnUuid.isPresent() ) {
-            Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get());
-            LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent());
-            if ( optInterVpnLink.isPresent() ) {
-                InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
-                String vpnUuid = optVpnUuid.get();
-                String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
-                if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid) ) {
-                    // This is an static route that points to the other endpoint of an InterVpnLink
-                    // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
-                    installRouteInInterVpnLink(interVpnLink, vpnUuid, vrfEntry, vpnId);
-                    installInterVpnRouteInLFib(rd, vrfEntry);
-                }
+        if (optVpnUuid.isPresent()) {
+            String vpnUuid = optVpnUuid.get();
+            InterVpnLinkDataComposite interVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
+            if (interVpnLink != null) {
+                LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
+                FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
+                    if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
+                        // This is an static route that points to the other endpoint of an InterVpnLink
+                        // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
+                        installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
+                        installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
+                    }
+                });
             }
         }
     }
 
-
     /*
       Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
       The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
       provided by ResourceBatchingManager
      */
-    private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
+    private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
+                                  final VrfEntry vrfEntry) {
         final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
 
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
         Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
-        Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
+        Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
+            + " has null vpnId!");
 
         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
-        final String rd = vrfTableKey.getRouteDistinguisher();
         if (vpnToDpnList != null) {
             for (VpnToDpnList vpnDpn : vpnToDpnList) {
                 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
@@ -389,74 +482,13 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
     }
 
-    // FIXME: Refactoring needed here.
-    //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
-    private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
-                                   int addOrRemove) {
-        Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
-        final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
-
-        String rd = vrfTableKey.getRouteDistinguisher();
-        VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
-            if (vpnInstance == null) {
-                LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
-                return;
-            }
-        } else {
-            Preconditions.checkNotNull(vpnInstance,
-                                       "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
-        }
-        String vpnUuid = vpnInstance.getVpnInstanceName();
-        Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
-                "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
-
-        // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
-        // there an interVpnLink for the involved vpn in order to make learn the new route to
-        // the other part of the inter-vpn-link.
-
-        // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
-        Optional<InterVpnLink> interVpnLink =
-            (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd)
-                                                  : FibUtil.getInterVpnLinkByRd(dataBroker, rd);
-        if ( !interVpnLink.isPresent() ) {
-            LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
-            return;
-        }
-
-        // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
-        // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
-        boolean proceed =
-            (addOrRemove == NwConstants.DEL_FLOW) || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
-                                                       && interVpnLink.get().isBgpRoutesLeaking() );
-
-        if ( proceed ) {
-            String theOtherVpnId = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid)
-                    ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
-                    : vpnUuid;
-
-            String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
-            String endpointIp = vrfEntry.getNextHopAddressList().get(0);
-
-            InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
-                    InstanceIdentifier.builder(FibEntries.class)
-                            .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
-                            .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
-                            .build();
-            if ( addOrRemove == NwConstants.ADD_FLOW ) {
-                LOG.debug("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
-                        vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
-                String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
-                long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
-                VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
-                        .setLabel(label)
-                        .setOrigin(RouteOrigin.INTERVPN.getValue())
-                        .build();
-                MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
-            } else {
-                LOG.debug("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
-                MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn);
-            }
+    void refreshFibTables(String rd, String prefix) {
+        InstanceIdentifier<VrfEntry> vrfEntryId =
+                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
+                        .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
+        Optional<VrfEntry> vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
+        if (vrfEntry.isPresent()) {
+            createFibEntries(vrfEntryId, vrfEntry.get());
         }
     }
 
@@ -468,495 +500,541 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         prefixBuilder.setIpAddress(lri.getPrefix());
         // Increment the refCount here
         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
-                .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
+            .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
         LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
         if (!isPresentInList) {
             LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
             List<String> vpnInstanceNames = lri.getVpnInstanceList();
             vpnInstanceNames.add(vpnInstanceName);
             builder.setVpnInstanceList(vpnInstanceNames);
-            FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
+            MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
         } else {
             LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
         }
         return prefixBuilder.build();
     }
 
-    private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
-                                         final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
+    void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
+                                         final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx) {
         Boolean wrTxPresent = true;
         if (tx == null) {
             wrTxPresent = false;
             tx = dataBroker.newWriteOnlyTransaction();
         }
-        synchronized (vrfEntry.getLabel().toString().intern()) {
-            LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-            if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
-                    vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-
-                if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
-                    Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
-                    if (vpnInstanceOpDataEntryOptional.isPresent()) {
-                        String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
-                        if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
-                            updateVpnReferencesInLri(lri, vpnInstanceName, false);
+        FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
+            List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+            synchronized (label.toString().intern()) {
+                LabelRouteInfo lri = getLabelRouteInfo(label);
+                if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
+
+                    if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
+                        Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
+                                FibUtil.getVpnInstanceOpData(dataBroker, rd);
+                        if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                            String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                            if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
+                                updateVpnReferencesInLri(lri, vpnInstanceName, false);
+                            }
                         }
                     }
+                    LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
+                            label, lri.getVpnInterfaceName(), lri.getDpnId());
+                }
+            }
+        });
+        final List<InstructionInfo> instructions = new ArrayList<>();
+        BigInteger subnetRouteMeta = ((BigInteger.valueOf(elanTag)).shiftLeft(24))
+            .or((BigInteger.valueOf(vpnId).shiftLeft(1)));
+        instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
+        instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
+        makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+
+        if (vrfEntry.getRoutePaths() != null) {
+            for (RoutePaths routePath : vrfEntry.getRoutePaths()) {
+                if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                    List<ActionInfo> actionsInfos = new ArrayList<>();
+                    // reinitialize instructions list for LFIB Table
+                    final List<InstructionInfo> LFIBinstructions = new ArrayList<>();
+
+                    actionsInfos.add(new ActionPopMpls());
+                    LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
+                    LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta,
+                            MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
+                    LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
+
+                    makeLFibTableEntry(dpnId, routePath.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY,
+                            NwConstants.ADD_FLOW, tx);
                 }
-                LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
-                        vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
             }
         }
-        final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
-        BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
-        instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
-        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
-        makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
-
-        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
-            List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
-            // reinitialize instructions list for LFIB Table
-            final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
-
-            actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
-            LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
-            LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
-            LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
-
-            makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
-        }
-        if (!wrTxPresent ) {
+        if (!wrTxPresent) {
             tx.submit();
         }
     }
 
-    private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
+    /*
+     * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
+     * LportDispatcher table (via table 80)
+     */
+    private void installInterVpnRouteInLFib(final InterVpnLinkDataComposite interVpnLink, final String vpnName,
+                                            final VrfEntry vrfEntry) {
         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
         // packet is commuted from Vpn2 to Vpn1.
-        Optional<String> vpnNameOpc = FibUtil.getVpnNameFromRd(dataBroker, rd);
-        if ( !vpnNameOpc.isPresent() ) {
-            LOG.warn("Could not find VpnInstanceName for Route-Distinguisher {}", rd);
+        String interVpnLinkName = interVpnLink.getInterVpnLinkName();
+        if (!interVpnLink.isActive()) {
+            LOG.warn("InterVpnLink {} is NOT ACTIVE. InterVpnLink flows for prefix={} wont be installed in LFIB",
+                     interVpnLinkName, vrfEntry.getDestPrefix());
             return;
         }
 
-        String vpnName = vpnNameOpc.get();
-        List<InterVpnLink> interVpnLinks = FibUtil.getAllInterVpnLinks(dataBroker);
-        boolean interVpnLinkFound = false;
-        for ( InterVpnLink interVpnLink : interVpnLinks ) {
-            boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
-            boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
-                    && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
-            if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
-                interVpnLinkFound = true;
-
-                Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName());
-                if ( !vpnLinkState.isPresent()
-                        || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
-                    LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
-                            interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
-                            interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
-                    return;
-                }
-
-                List<BigInteger> targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId()
-                                                               : vpnLinkState.get().getSecondEndpointState().getDpId();
-                Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
-                                                 : vpnLinkState.get().getFirstEndpointState().getLportTag();
-
-                for ( BigInteger dpId : targetDpns ) {
-                    List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
-
-                    BigInteger[] metadata = new BigInteger[] {
-                            MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
-                            MetaDataUtil.getMetaDataMaskForLPortDispatcher()
-                    };
-                    List<InstructionInfo> instructions =
-                            Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
-                                    new InstructionInfo(InstructionType.write_metadata, metadata),
-                                    new InstructionInfo(InstructionType.goto_table,
-                                            new long[] { NwConstants.L3_INTERFACE_TABLE }));
-
-                    makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
-                            NwConstants.ADD_FLOW, null);
-                }
+        List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
+        Optional<Long> optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName);
+        if (!optLportTag.isPresent()) {
+            LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName);
+            return;
+        }
 
-                break;
-            }
+        Long lportTag = optLportTag.get();
+        Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null);
+        if (label == null) {
+            LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped",
+                      vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths());
+            return;
         }
+        List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
+        List<InstructionInfo> instructions = Arrays.asList(
+            new InstructionApplyActions(actionsInfos),
+            new InstructionWriteMetadata(MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
+                                                            ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
+                                                                                  NwConstants.L3VPN_SERVICE_INDEX)),
+                                         MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
+            new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
+        List<String> interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+
+        for (BigInteger dpId : targetDpns) {
+            LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
+                      vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName());
 
-        if ( !interVpnLinkFound ) {
-            LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
-                    vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
+            makeLFibTableEntry(dpId, label, instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW,
+                               /*writeTx*/null);
         }
     }
 
 
-
-    private void installRouteInInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
-                                            final VrfEntry vrfEntry, long vpnTag) {
+    /*
+     * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
+     */
+    private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
+                                               final VrfEntry vrfEntry, long vpnTag) {
         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
-        Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
-                && vrfEntry.getNextHopAddressList().size() == 1);
+        Preconditions.checkArgument(vrfEntry.getRoutePaths() != null
+            && vrfEntry.getRoutePaths().size() == 1);
         String destination = vrfEntry.getDestPrefix();
-        String nextHop = vrfEntry.getNextHopAddressList().get(0);
-        String iVpnLinkName = interVpnLink.getInterVpnLinkName();
+        String nextHop = vrfEntry.getRoutePaths().get(0).getNexthopAddress();
+        String interVpnLinkName = interVpnLink.getInterVpnLinkName();
 
         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
         // using as metadata the LPortTag associated to that vpn in the inter-vpn-link.
-        if ( interVpnLink.getState().or(State.Error) != State.Active ) {
+        if (interVpnLink.getState().or(State.Error) != State.Active) {
             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
-                    destination, nextHop, iVpnLinkName);
+                destination, nextHop, interVpnLinkName);
             return;
         }
 
         Optional<Long> optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid);
-        if ( !optOtherEndpointLportTag.isPresent() ) {
+        if (!optOtherEndpointLportTag.isPresent()) {
             LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}",
-                     vpnUuid, iVpnLinkName);
+                vpnUuid, interVpnLinkName);
             return;
         }
 
         List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid);
-        if ( targetDpns.isEmpty() ) {
-            LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, iVpnLinkName);
+        if (targetDpns.isEmpty()) {
+            LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}",
+                vpnUuid, interVpnLinkName);
             return;
         }
 
-        BigInteger[] metadata = new BigInteger[] {
-                MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
-                        ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
-                MetaDataUtil.getMetaDataMaskForLPortDispatcher()
-        };
-        List<Instruction> instructions =
-                Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
-                              new InstructionInfo(InstructionType.goto_table,
-                                                  new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
-
-        String values[] = destination.split("/");
+        String[] values = destination.split("/");
         String destPrefixIpAddress = values[0];
         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 
         List<MatchInfo> matches = new ArrayList<>();
-        matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
-                                                                              MetaDataUtil.METADATA_MASK_VRFID }));
-        matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
+        matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
+        matches.add(MatchEthernetType.IPV4);
 
         if (prefixLength != 0) {
-            matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
-                                      new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
+            matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
         }
 
+        List<Instruction> instructions =
+            Arrays.asList(new InstructionWriteMetadata(
+                    MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
+                        ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
+                            .L3VPN_SERVICE_INDEX)),
+                    MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
+                new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
+
         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
-        String flowRef = getInterVpnFibFlowRef(iVpnLinkName, destination, nextHop);
+        String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop);
         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
-                                                 COOKIE_VM_FIB_TABLE, matches, instructions);
-
-        for ( BigInteger dpId : targetDpns ) {
-            mdsalManager.installFlow(dpId, flowEntity);
-        }
-    }
-
-    private void removeRouteFromInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
-                                             final VrfEntry vrfEntry) {
-
-        Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
-        Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
-                                    && vrfEntry.getNextHopAddressList().size() == 1);
-
-        String iVpnLinkName = interVpnLink.getInterVpnLinkName();
-
-        InterVpnLinkState interVpnLinkState = interVpnLink.getInterVpnLinkState();
-        if ( interVpnLinkState == null ) {
-            LOG.warn("Could not find State for InterVpnLink {}", iVpnLinkName);
-            return;
-        }
-
-        String nextHop = vrfEntry.getNextHopAddressList().get(0);
-        String flowRef = getInterVpnFibFlowRef(iVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
-        FlowId flowId = new FlowId(flowRef);
-        Flow flow = new FlowBuilder().setKey(new FlowKey(flowId)).setId(flowId).setTableId(NwConstants.L3_FIB_TABLE)
-                                     .setFlowName(flowRef).build();
-
-        for ( BigInteger dpId : interVpnLink.getEndpointDpnsByVpnName(vpnUuid) ) {
-            mdsalManager.removeFlow(dpId, flow);
-        }
+            COOKIE_VM_FIB_TABLE, matches, instructions);
 
-    }
+        LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}",
+            vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 
-    private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
-                                                     InstanceIdentifier<T> path) {
+        for (BigInteger dpId : targetDpns) {
 
-        ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
+            LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB",
+                vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(),
+                dpId, interVpnLink.getInterVpnLinkName());
 
-        Optional<T> result = Optional.absent();
-        try {
-            result = tx.read(datastoreType, path).get();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
+            mdsalManager.installFlow(dpId, flowEntity);
         }
-
-        return result;
     }
 
-    private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
+    private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
         List<BigInteger> returnLocalDpnId = new ArrayList<>();
-        Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
-        String localNextHopIP = vrfEntry.getDestPrefix();
+        Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
 
         if (localNextHopInfo == null) {
             //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
-            Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            if (extraRoute != null) {
+            Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
+                    FibUtil.getVpnNameFromId(dataBroker, vpnId), rd, vrfEntry.getDestPrefix());
+            if (extraRouteOptional.isPresent()) {
+                Routes extraRoute = extraRouteOptional.get();
                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
                     if (nextHopIp != null) {
-                        localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
-                        localNextHopIP = nextHopIp + "/32";
-                        BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
-                        returnLocalDpnId.add(dpnId);
+                        localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp
+                                + NwConstants.IPV4PREFIX);
+                        if (localNextHopInfo != null) {
+                            returnLocalDpnId.add(localNextHopInfo.getDpnId());
+                        }
                     }
                 }
             }
+        } else {
+            returnLocalDpnId.add(localNextHopInfo.getDpnId());
+        }
+
+        return returnLocalDpnId;
+    }
+
+    private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
+        List<BigInteger> returnLocalDpnId = new ArrayList<>();
+        Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
+        String localNextHopIP = vrfEntry.getDestPrefix();
+        String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
+        if (localNextHopInfo == null) {
+            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
+            List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
+                    vpnName, usedRds, localNextHopIP);
+            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+            vpnExtraRoutes.stream().forEach(extraRoute -> {
+                Prefixes localNextHopInfoLocal = FibUtil.getPrefixToInterface(dataBroker,
+                        vpnId, extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
+                BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(),
+                        vpnId, rd, vrfEntry, vpnId, extraRoute, vpnExtraRoutes);
+                returnLocalDpnId.add(dpnId);
+            });
             if (localNextHopInfo == null) {
             /* imported routes case */
-                synchronized (vrfEntry.getLabel().toString().intern()) {
-                    LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-                    if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
-                            vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-                        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
-                            Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
-                            if (vpnInstanceOpDataEntryOptional.isPresent()) {
-                                String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
-                                if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
-                                    localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
-                                    localNextHopIP = lri.getPrefix();
-                                } else {
-                                    localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
-                                    localNextHopIP = lri.getPrefix();
+                if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
+                    java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+                    if (optionalLabel.isPresent()) {
+                        Long label = optionalLabel.get();
+                        List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+                        synchronized (label.toString().intern()) {
+                            LabelRouteInfo lri = getLabelRouteInfo(label);
+                            if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
+                                Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
+                                        FibUtil.getVpnInstanceOpData(dataBroker, rd);
+                                if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                                    String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                                    if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
+                                        localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
+                                        localNextHopIP = lri.getPrefix();
+                                    } else {
+                                        localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
+                                        localNextHopIP = lri.getPrefix();
+                                    }
+                                }
+                                if (localNextHopInfo != null) {
+                                    LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
+                                            label, localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
+                                    if (vpnExtraRoutes == null || vpnExtraRoutes.isEmpty()) {
+                                        BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP,
+                                                vpnId, rd, vrfEntry, lri.getParentVpnid(), null, vpnExtraRoutes);
+                                        returnLocalDpnId.add(dpnId);
+                                    } else {
+                                        for (Routes extraRoutes : vpnExtraRoutes) {
+                                            BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo,
+                                                    localNextHopIP,
+                                                    vpnId, rd, vrfEntry, lri.getParentVpnid(),
+                                                    extraRoutes, vpnExtraRoutes);
+                                            returnLocalDpnId.add(dpnId);
+                                        }
+                                    }
                                 }
-                            }
-                            if (localNextHopInfo != null) {
-                                LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
-                                        vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
-                                BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
-                                returnLocalDpnId.add(dpnId);
                             }
                         }
                     }
                 }
             }
+            if (returnLocalDpnId.isEmpty()) {
+                LOG.error("Local DPNID is empty for rd {}, vpnId {}, vrfEntry {}", rd, vpnId, vrfEntry);
+            }
         } else {
-            BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
+            BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId,
+                    rd, vrfEntry, vpnId, /*routes*/ null, /*vpnExtraRoutes*/ null);
             returnLocalDpnId.add(dpnId);
         }
-
         return returnLocalDpnId;
     }
 
-    private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
-                                                final VrfEntry vrfEntry, Long parentVpnId){
+    private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP,
+                                                final Long vpnId, final String rd,
+                                                final VrfEntry vrfEntry, Long parentVpnId,
+                                                Routes routes, List<Routes> vpnExtraRoutes) {
+        String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
         if (localNextHopInfo != null) {
+            long groupId;
+            long localGroupId;
             final BigInteger dpnId = localNextHopInfo.getDpnId();
-            if (!isVpnPresentInDpn(rd, dpnId)) {
-                LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString());
-                return BigInteger.ZERO;
+            if (Boolean.TRUE.equals(localNextHopInfo.isNatPrefix())) {
+                LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip local dpn {} FIB processing",
+                        vrfEntry.getDestPrefix(), vpnId, rd, dpnId);
+                return dpnId;
             }
-
-            final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
-            if (groupId == 0) {
+            String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix());
+            String interfaceName = localNextHopInfo.getVpnInterfaceName();
+            String prefix = vrfEntry.getDestPrefix();
+            String gwMacAddress = vrfEntry.getGatewayMacAddress();
+            //The loadbalancing group is created only if the extra route has multiple nexthops
+            //to avoid loadbalancing the discovered routes
+            if (routes != null && (vpnExtraRoutes.size() > 1 || routes.getNexthopIpList().size() > 1)) {
+                localNextHopIP = routes.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX;
+                groupId = nextHopManager.createNextHopGroups(parentVpnId, rd, dpnId, vrfEntry, routes,
+                        vpnExtraRoutes);
+                localGroupId = nextHopManager.getLocalNextHopGroup(parentVpnId, localNextHopIP);
+            } else {
+                groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, interfaceName, localNextHopIP, prefix,
+                        gwMacAddress, jobKey);
+                localGroupId = groupId;
+            }
+            if (groupId == FibConstants.INVALID_GROUP_ID) {
                 LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}",
-                        vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
+                    prefix, rd, interfaceName, dpnId.toString());
                 return BigInteger.ZERO;
             }
-            List<ActionInfo> actionsInfos =
-                    Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
-            final List<InstructionInfo> instructions =
-                    Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
-            actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
-                    new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
-            final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
-            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
-                LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
-                        dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
-            } else {
-                LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
-            }
-            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-                            makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
-                            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
-                                makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
-                                makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
-                            }
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            futures.add(tx.submit());
-                            return futures;
+            final List<InstructionInfo> instructions = Collections.singletonList(
+                new InstructionApplyActions(
+                    Collections.singletonList(new ActionGroup(groupId))));
+            final List<InstructionInfo> lfibinstructions = Collections.singletonList(
+                new InstructionApplyActions(
+                    Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
+            java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+            List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+            if (!FibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(dataBroker, localNextHopInfo
+                    .getSubnetId(), vpnName, rd)) {
+                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+                dataStoreCoordinator.enqueueJob(jobKey, () -> {
+                    WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+                    makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+                    optLabel.ifPresent(label -> {
+                        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                            LOG.debug("Installing LFIB and tunnel table entry on dpn {} for interface {} with label "
+                                            + "{}, rd {}, prefix {}, nexthop {}", dpnId,
+                                    localNextHopInfo.getVpnInterfaceName(), optLabel, rd, vrfEntry.getDestPrefix(),
+                                    nextHopAddressList);
+                            makeLFibTableEntry(dpnId, label, lfibinstructions, DEFAULT_FIB_FLOW_PRIORITY,
+                                    NwConstants.ADD_FLOW, tx);
+                            makeTunnelTableEntry(dpnId, label, localGroupId, tx);
+                        } else {
+                            LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. "
+                                            + "LFib and Terminating table entries will not be created.",
+                                    rd, vrfEntry.getDestPrefix(), optLabel, nextHopAddressList, vpnId);
                         }
                     });
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    futures.add(tx.submit());
+                    return futures;
+                });
+            }
             return dpnId;
         }
+        LOG.error("localNextHopInfo received is null for prefix {} on rd {} on vpn {}",
+                vrfEntry.getDestPrefix(), rd, vpnName);
         return BigInteger.ZERO;
     }
 
-    private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
+    private boolean isVpnPresentInDpn(String rd, BigInteger dpnId) {
         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
-        Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
-        if (dpnInVpn.isPresent()) {
-            return true;
-        }
-        return false;
+        return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id).isPresent();
     }
 
     private LabelRouteInfo getLabelRouteInfo(Long label) {
-        InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
-                .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
-        Optional<LabelRouteInfo> opResult = read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
+        InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
+            .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
+        Optional<LabelRouteInfo> opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
         if (opResult.isPresent()) {
             return opResult.get();
         }
         return null;
     }
 
-    private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
+    private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName, WriteTransaction tx) {
         LOG.debug("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
         InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
-                .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
+            .child(LabelRouteInfo.class, new LabelRouteInfoKey(lri.getLabel())).build();
         if (lri == null) {
             return true;
         }
-        List<String> vpnInstancesList = lri.getVpnInstanceList() != null ? lri.getVpnInstanceList() : new ArrayList<String>();
+        List<String> vpnInstancesList = lri.getVpnInstanceList() != null
+            ? lri.getVpnInstanceList() : new ArrayList<>();
         if (vpnInstancesList.contains(vpnInstanceName)) {
             LOG.debug("vpninstance {} name is present", vpnInstanceName);
             vpnInstancesList.remove(vpnInstanceName);
         }
         if (vpnInstancesList.size() == 0) {
             LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
-            FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
+            if (tx != null) {
+                tx.delete(LogicalDatastoreType.OPERATIONAL, lriId);
+            } else {
+                MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId);
+            }
             return true;
         } else {
             LOG.debug("updating LRI instance object for label {}", lri.getLabel());
             LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
-            FibUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
+            MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build());
         }
         return false;
     }
 
-    private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
+    void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
                                       WriteTransaction tx) {
-        List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
-        actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
+        List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 
-
-        createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
+        createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx);
 
         LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
-                dpId, label, groupId);
+            dpId, label, groupId);
     }
 
-    public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
-                                                 WriteTransaction tx) {
+    public void createTerminatingServiceActions(BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
+                                                WriteTransaction tx) {
         List<MatchInfo> mkMatches = new ArrayList<>();
 
-        LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
+        LOG.debug("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}",
+            destDpId, label, actionsInfos);
 
         // Matching metadata
         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
-        mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+        mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 
         List<InstructionInfo> mkInstructions = new ArrayList<>();
-        mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
+        mkInstructions.add(new InstructionApplyActions(actionsInfos));
 
-        FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
-                getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
-                0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
+        FlowEntity terminatingServiceTableFlowEntity =
+            MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
+            getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5,
+                String.format("%s:%d", "TST Flow Entry ", label),
+            0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions);
 
-        FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
+        FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId()));
 
         FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 
         Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
-        tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
+            .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+            .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId()))
+            .child(Flow.class, flowKey).build();
+        tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(), true);
     }
 
     private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
         FlowEntity flowEntity;
-        LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
+        LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label);
         List<MatchInfo> mkMatches = new ArrayList<>();
         // Matching metadata
-        mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+        mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
         flowEntity = MDSALUtil.buildFlowEntity(dpId,
-                NwConstants.INTERNAL_TUNNEL_TABLE,
-                getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
-                5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
-                COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
+            NwConstants.INTERNAL_TUNNEL_TABLE,
+            getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label),
+            5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0,
+            COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
         Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
         FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
+            .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+            .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 
         tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
         LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
     }
 
-    /**
-     * Delete local FIB entry
-     * @param vpnId
-     * @param rd
-     * @param vrfEntry
-     * @return
-     */
     public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
         List<BigInteger> returnLocalDpnId = new ArrayList<>();
-        VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
+        Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
         String localNextHopIP = vrfEntry.getDestPrefix();
+        String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
 
         if (localNextHopInfo == null) {
-            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
-            Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            if (extra_route != null) {
-                for (String nextHopIp : extra_route.getNexthopIpList()) {
-                    LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
-                    if (nextHopIp != null) {
-                        localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
-                        localNextHopIP = nextHopIp + "/32";
-                        BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
-                                vpnId, rd, vrfEntry, true /*isExtraRoute*/);
-                        if (!dpnId.equals(BigInteger.ZERO)) {
-                            returnLocalDpnId.add(dpnId);
-                        }
+            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
+            if (usedRds.size() > 1) {
+                LOG.error("The extra route prefix {} is still present in some DPNs in vpn {} on rd {}",
+                        vrfEntry.getDestPrefix(), vpnName, rd);
+                return returnLocalDpnId;
+            }
+            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency
+            //in the vpn
+            Optional<Routes> extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
+                    vpnName, rd, vrfEntry.getDestPrefix());
+            if (extraRouteOptional.isPresent()) {
+                Routes extraRoute = extraRouteOptional.get();
+                localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId,
+                        extraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX);
+                if (localNextHopInfo != null) {
+                    BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
+                            vpnId, rd, vrfEntry);
+                    if (!dpnId.equals(BigInteger.ZERO)) {
+                        nextHopManager.setupLoadBalancingNextHop(vpnId, dpnId,
+                                vrfEntry.getDestPrefix(), /*listBucketInfo*/ null, /*remove*/ false);
+                        returnLocalDpnId.add(dpnId);
                     }
+                } else {
+                    LOG.error("localNextHopInfo unavailable while deleting prefix {} with rds {}, primary rd {} in "
+                            + "vpn {}", vrfEntry.getDestPrefix(), usedRds, rd, vpnName);
                 }
             }
 
             if (localNextHopInfo == null) {
-              /* Imported VRF entry */
-                LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-                if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
-                        vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-                    VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
-                    vpnNexthopBuilder.setDpnId(lri.getDpnId());
-                    BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
-                            vpnId, rd, vrfEntry, false /*isExtraRoute*/);
-                    if (!dpnId.equals(BigInteger.ZERO)) {
-                        returnLocalDpnId.add(dpnId);
+                /* Imported VRF entry */
+                java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+                if (optionalLabel.isPresent()) {
+                    Long label = optionalLabel.get();
+                    List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+                    LabelRouteInfo lri = getLabelRouteInfo(label);
+                    if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
+                        PrefixesBuilder prefixBuilder = new PrefixesBuilder();
+                        prefixBuilder.setDpnId(lri.getDpnId());
+                        BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), localNextHopIP,
+                                vpnId, rd, vrfEntry);
+                        if (!dpnId.equals(BigInteger.ZERO)) {
+                            returnLocalDpnId.add(dpnId);
+                        }
                     }
                 }
             }
 
-
         } else {
             BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
-                    vpnId, rd, vrfEntry, false /*isExtraRoute*/);
+                vpnId, rd, vrfEntry);
             if (!dpnId.equals(BigInteger.ZERO)) {
                 returnLocalDpnId.add(dpnId);
             }
@@ -965,139 +1043,214 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         return returnLocalDpnId;
     }
 
-    private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
+    private BigInteger checkDeleteLocalFibEntry(Prefixes localNextHopInfo, final String localNextHopIP,
                                                 final Long vpnId, final String rd,
-                                                final VrfEntry vrfEntry, final boolean isExtraRoute) {
+                                                final VrfEntry vrfEntry) {
         if (localNextHopInfo != null) {
-            final BigInteger dpnId = localNextHopInfo.getDpnId();;
-            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-                            makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
-                                    NwConstants.DEL_FLOW, tx);
-                            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
-                                makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
-                                        DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
-                                removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
-                            }
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            futures.add(tx.submit());
-                            return futures;
+            final BigInteger dpnId = localNextHopInfo.getDpnId();
+            if (!FibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(dataBroker, localNextHopInfo
+                    .getSubnetId(), vpnId, rd)) {
+                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+                dataStoreCoordinator.enqueueJob("FIB-" + vpnId.toString() + "-" + dpnId.toString() + "-" + vrfEntry
+                        .getDestPrefix(), () -> {
+                        WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+                        makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */, NwConstants.DEL_FLOW,
+                              tx);
+                        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                            FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
+                                makeLFibTableEntry(dpnId, label, null /* instructions */, DEFAULT_FIB_FLOW_PRIORITY,
+                                        NwConstants.DEL_FLOW, tx);
+                                removeTunnelTableEntry(dpnId, label, tx);
+                            });
                         }
+                        List<ListenableFuture<Void>> futures = new ArrayList<>();
+                        futures.add(tx.submit());
+                        return futures;
                     });
+            }
             //TODO: verify below adjacency call need to be optimized (?)
-            deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
+            deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix());
             return dpnId;
         }
         return BigInteger.ZERO;
     }
 
-
-    private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
-        return InstanceIdentifier.builder(PrefixToInterface.class)
-                .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
-    }
-
-    private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
-        Optional<Prefixes> localNextHopInfoData =
-                FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
-        return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
-    }
-
-    private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
-        return InstanceIdentifier.builder(VpnToExtraroute.class)
-                .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
-                        new ExtrarouteKey(ipPrefix)).build();
-    }
-
-    private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
-        Optional<Extraroute> extraRouteInfo =
-                FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
-        return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
-
+    static  InstanceIdentifier<Routes> getVpnToExtrarouteIdentifier(String vpnName, String vrfId,
+            String ipPrefix) {
+        return InstanceIdentifier.builder(VpnToExtraroutes.class)
+                .child(Vpn.class, new VpnKey(vpnName)).child(ExtraRoutes.class,
+                        new ExtraRoutesKey(vrfId)).child(Routes.class, new RoutesKey(ipPrefix)).build();
     }
 
-    private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
-        try {
-            Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
-                    new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
-            RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
-            if(!rpcResult.isSuccessful()) {
-                LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
-            } else {
-                return rpcResult.getResult().getTunnelType();
-            }
-
-        } catch (InterruptedException | ExecutionException e) {
-            LOG.warn("Exception when getting tunnel interface Id for tunnel type", e);
+    public Routes getVpnToExtraroute(Long vpnId, String vpnRd, String destPrefix) {
+        String optVpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
+        if (optVpnName != null) {
+            InstanceIdentifier<Routes> vpnExtraRoutesId = getVpnToExtrarouteIdentifier(
+                    optVpnName, vpnRd, destPrefix);
+            return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId).orNull();
         }
-
         return null;
-
     }
+
     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
-                                      final VrfEntry vrfEntry, WriteTransaction tx) {
+            final VrfEntry vrfEntry, WriteTransaction tx) {
         Boolean wrTxPresent = true;
         if (tx == null) {
             wrTxPresent = false;
             tx = dataBroker.newWriteOnlyTransaction();
         }
         String rd = vrfTableKey.getRouteDistinguisher();
-        LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
-                vrfEntry.getDestPrefix(), rd, tx);
-        /********************************************/
-        List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
-
-        if (tunnelInterfaceList.isEmpty()) {
-            LOG.error("Could not get interface for nexthop: {} in vpn {}",
-                    vrfEntry.getNextHopAddressList(), rd);
-            LOG.warn("Failed to add Route: {} in vpn: {}",
-                    vrfEntry.getDestPrefix(), rd);
+        String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnId);
+        LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
+                vrfEntry.getDestPrefix(), rd, remoteDpnId);
+
+        List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
+        if (adjacencyResults == null || adjacencyResults.isEmpty()) {
+            LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
+            LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
             return;
         }
-
-        for (String tunnelInterface : tunnelInterfaceList) {
-            List<InstructionInfo> instructions = new ArrayList<>();
-            List<ActionInfo> actionInfos = new ArrayList<>();
-            Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
-            if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
-                LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
-                actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
-                actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
-            } else {
-                int label = vrfEntry.getLabel().intValue();
-                BigInteger tunnelId;
-                // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
-                if (tunnel_type.equals(TunnelTypeVxlan.class)) {
-                    tunnelId = BigInteger.valueOf(label);
-                } else {
-                    tunnelId = BigInteger.valueOf(label);
+        if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
+            programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
+        } else {
+            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
+            List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
+                    vpnName, usedRds, vrfEntry.getDestPrefix());
+            if (!vpnExtraRoutes.isEmpty()) {
+                List<InstructionInfo> instructions = new ArrayList<>();
+                long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry,
+                        null, vpnExtraRoutes);
+                if (groupId == FibConstants.INVALID_GROUP_ID) {
+                    LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}",
+                            vrfEntry.getDestPrefix(), rd, remoteDpnId.toString());
+                    return;
                 }
-
-                LOG.debug("adding set tunnel id action for label {}", label);
-                actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
+                List<ActionInfo> actionInfos =
+                        Collections.singletonList(new ActionGroup(groupId));
+                instructions.add(new InstructionApplyActions(actionInfos));
+                makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+            } else {
+                programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
+            }
+        }
+        if (!wrTxPresent) {
+            tx.submit();
+        }
+        LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
+    }
+
+    private void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
+            final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
+        List<InstructionInfo> instructions = new ArrayList<>();
+        for (AdjacencyResult adjacencyResult : adjacencyResults) {
+            List<ActionInfo> actionInfos = new ArrayList<>();
+            String egressInterface = adjacencyResult.getInterfaceName();
+            if (FibUtil.isTunnelInterface(adjacencyResult)) {
+                addTunnelInterfaceActions(adjacencyResult, vpnId, vrfEntry, actionInfos, rd);
+            } else {
+                addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
             }
-            List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
-            if(egressActions.isEmpty()){
-                LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
+            List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface,
+                    actionInfos.size());
+            if (egressActions.isEmpty()) {
+                LOG.error(
+                        "Failed to retrieve egress action for prefix {} route-paths {} interface {}. "
+                                + "Aborting remote FIB entry creation.",
+                                vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
                 return;
             }
             actionInfos.addAll(egressActions);
-            instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
-            makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+            instructions.add(new InstructionApplyActions(actionInfos));
         }
-        if(!wrTxPresent ){
-            tx.submit();
+        makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+    }
+
+    private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
+        if (vrfEntry.getMac() != null) {
+            actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
+                new MacAddress(vrfEntry.getMac())));
+            return;
+        }
+
+        String ipPrefix = vrfEntry.getDestPrefix();
+        Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
+        if (prefixInfo == null) {
+            LOG.debug("No prefix info found for prefix {}", ipPrefix);
+            return;
+        }
+
+        String ifName = prefixInfo.getVpnInterfaceName();
+        if (ifName == null) {
+            LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
+            return;
+        }
+
+        String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
+        if (macAddress == null) {
+            LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
+            return;
+        }
+
+        actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
+    }
+
+    private void addTunnelInterfaceActions(AdjacencyResult adjacencyResult, long vpnId, VrfEntry vrfEntry,
+                                           List<ActionInfo> actionInfos, String rd) {
+        Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(interfaceManager,
+                adjacencyResult.getInterfaceName());
+        // TODO - For now have added routePath into adjacencyResult so that we know for which
+        // routePath this result is built for. If this is not possible construct a map which does
+        // the same.
+        String nextHopIp = adjacencyResult.getNextHopIp();
+        java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
+        if (!optionalLabel.isPresent()) {
+            LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
+            return;
+        }
+        long label = optionalLabel.get();
+        if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
+            LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
+            actionInfos.add(new ActionPushMpls());
+            actionInfos.add(new ActionSetFieldMplsLabel(label));
+            actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
+        } else {
+            BigInteger tunnelId = null;
+            // FIXME vxlan vni bit set is not working properly with OVS.need to
+            // revisit
+            if (tunnelType.equals(TunnelTypeVxlan.class)) {
+                Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
+                // Internet VPN VNI will be used as tun_id for NAT use-cases
+                if (prefixInfo.isNatPrefix()) {
+                    if (vrfEntry.getL3vni() != null && vrfEntry.getL3vni() != 0) {
+                        tunnelId = BigInteger.valueOf(vrfEntry.getL3vni());
+                    }
+                } else {
+                    if (FibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(dataBroker, prefixInfo.getSubnetId(),
+                            vpnId, rd)) {
+                        java.util.Optional<Long> optionalVni = FibUtil.getVniForVxlanNetwork(dataBroker,
+                                prefixInfo.getSubnetId());
+                        if (!optionalVni.isPresent()) {
+                            LOG.error("VNI not found for nexthop {} vrfEntry {} with subnetId {}", nextHopIp,
+                                    vrfEntry, prefixInfo.getSubnetId());
+                            return;
+                        }
+                        tunnelId = BigInteger.valueOf(optionalVni.get());
+                    } else {
+                        tunnelId = BigInteger.valueOf(label);
+                    }
+                }
+            } else {
+                tunnelId = BigInteger.valueOf(label);
+            }
+            LOG.debug("adding set tunnel id action for label {}", label);
+            actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
+            addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
         }
-        LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
     }
 
     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
-        Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
+        Optional<VpnToDpnList> dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
         if (dpnInVpn.isPresent()) {
             List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
             VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
@@ -1105,19 +1258,19 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             if (vpnInterfaces.remove(currVpnInterface)) {
                 if (vpnInterfaces.isEmpty()) {
                     LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
-                    FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
+                    MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
                     cleanUpDpnForVpn(dpnId, vpnId, rd, null);
                 } else {
                     LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
-                    FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
-                            VpnInterfaces.class,
-                            new VpnInterfacesKey(intfName)));
+                    MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id.child(
+                        VpnInterfaces.class,
+                        new VpnInterfacesKey(intfName)));
                 }
             }
         }
     }
 
-    private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
+    void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) {
     /* Get interface info from prefix to interface mapping;
         Use the interface info to get the corresponding vpn interface op DS entry,
         remove the adjacency corresponding to this fib entry.
@@ -1127,51 +1280,63 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
          - vpn interface op DS
      */
         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
-        Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
-        Extraroute extraRoute = null;
+        Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
+        Routes extraRoute = null;
         if (prefixInfo == null) {
-            extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            if(extraRoute != null) {
+            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
+            String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0);
+            extraRoute = getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix());
+            if (extraRoute != null) {
                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
-
                     if (nextHopIp != null) {
-                        prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
-                        checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
+                        prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp
+                                + NwConstants.IPV4PREFIX);
+                        checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
                     }
                 }
             }
             if (prefixInfo == null) {
-                LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-                if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
-                        vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-                    PrefixesBuilder prefixBuilder = new PrefixesBuilder();
-                    prefixBuilder.setDpnId(lri.getDpnId());
-                    prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
-                    prefixBuilder.setIpAddress(lri.getPrefix());
-                    prefixInfo = prefixBuilder.build();
-                    LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
-                            vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
-                    checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
+                java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+                if (optionalLabel.isPresent()) {
+                    Long label = optionalLabel.get();
+                    List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+                    LabelRouteInfo lri = getLabelRouteInfo(label);
+                    if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
+                        PrefixesBuilder prefixBuilder = new PrefixesBuilder();
+                        prefixBuilder.setDpnId(lri.getDpnId());
+                        prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
+                        prefixBuilder.setIpAddress(lri.getPrefix());
+                        prefixInfo = prefixBuilder.build();
+                        LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
+                                label, prefixInfo.getVpnInterfaceName(), lri.getDpnId());
+                        checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
+                    }
                 }
             }
         } else {
-            checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
+            checkCleanUpOpDataForFib(prefixInfo, vpnId, primaryRd, vrfEntry, extraRoute);
         }
     }
 
     private void checkCleanUpOpDataForFib(final Prefixes prefixInfo, final Long vpnId, final String rd,
-                                          final VrfEntry vrfEntry, final Extraroute extraRoute) {
+                                          final VrfEntry vrfEntry, final Routes extraRoute) {
 
         if (prefixInfo == null) {
             LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
             return; //Don't have any info for this prefix (shouldn't happen); need to return
         }
 
+        if (Boolean.TRUE.equals(prefixInfo.isNatPrefix())) {
+            LOG.debug("NAT Prefix {} with vpnId {} rd {}. Skip FIB processing",
+                    vrfEntry.getDestPrefix(), vpnId, rd);
+            return;
+        }
+
         String ifName = prefixInfo.getVpnInterfaceName();
         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
         dataStoreCoordinator.enqueueJob("VPNINTERFACE-" + ifName,
-                new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
+            new CleanupVpnInterfaceWorker(prefixInfo, vpnId, rd, vrfEntry, extraRoute));
     }
 
     private class CleanupVpnInterfaceWorker implements Callable<List<ListenableFuture<Void>>> {
@@ -1179,14 +1344,14 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         Long vpnId;
         String rd;
         VrfEntry vrfEntry;
-        Extraroute extraRoute;
+        Routes extraRoute;
 
-        public CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
-                                         final VrfEntry vrfEntry, final Extraroute extraRoute) {
+        CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd,
+                                         final VrfEntry vrfEntry, final Routes extraRoute) {
             this.prefixInfo = prefixInfo;
             this.vpnId = vpnId;
-            this.rd= rd;
-            this.vrfEntry= vrfEntry;
+            this.rd = rd;
+            this.vrfEntry = vrfEntry;
             this.extraRoute = extraRoute;
         }
 
@@ -1194,72 +1359,90 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         public List<ListenableFuture<Void>> call() throws Exception {
             // If another renderer(for eg : CSS) needs to be supported, check can be performed here
             // to call the respective helpers.
+            WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 
             //First Cleanup LabelRouteInfo
-            synchronized (vrfEntry.getLabel().toString().intern()) {
-                LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-                if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
-                                vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-                    Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
-                    String vpnInstanceName = "";
-                    if (vpnInstanceOpDataEntryOptional.isPresent()) {
-                            vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
-                        }
-                    boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName);
-                    if (lriRemoved) {
-                            String parentRd = lri.getParentVpnRd();
+            //TODO(KIRAN) : Move the below block when addressing iRT/eRT for L3VPN Over VxLan
+            if (vrfEntry.getEncapType().equals(VrfEntry.EncapType.Mplsgre)) {
+                FibUtil.getLabelFromRoutePaths(vrfEntry).ifPresent(label -> {
+                    List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+                    synchronized (label.toString().intern()) {
+                        LabelRouteInfo lri = getLabelRouteInfo(label);
+                        if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
+                                && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) {
+                            Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
+                                    FibUtil.getVpnInstanceOpData(dataBroker, rd);
+                            String vpnInstanceName = "";
+                            if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                                vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                            }
+                            boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName, writeOperTxn);
+                            if (lriRemoved) {
+                                String parentRd = lri.getParentVpnRd();
+                                FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
+                                        FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
+                            }
+                        } else {
                             FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
-                                            FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
-                        }
-                } else {
-                    FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
                                     FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
-                }
+                        }
+                    }
+                });
             }
             String ifName = prefixInfo.getVpnInterfaceName();
-            Optional<VpnInterface> optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                    FibUtil.getVpnInterfaceIdentifier(ifName));
+            Optional<VpnInterface> optvpnInterface = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
+                FibUtil.getVpnInterfaceIdentifier(ifName));
             if (optvpnInterface.isPresent()) {
                 long associatedVpnId = FibUtil.getVpnId(dataBroker, optvpnInterface.get().getVpnInstanceName());
                 if (vpnId != associatedVpnId) {
                     LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
-                            vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
+                        vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
                     LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
                     return null;
                 } else {
                     LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
-                            vrfEntry.getDestPrefix(), associatedVpnId);
+                        vrfEntry.getDestPrefix(), associatedVpnId);
                 }
             }
             if (extraRoute != null) {
-                FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                        FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
+                Optional<String> optVpnName = FibUtil.getVpnNameFromRd(dataBroker, rd);
+                List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
+                //Only one used Rd present in case of removal event
+                String usedRd = usedRds.get(0);
+                if (optVpnName.isPresent()) {
+                    writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
+                            getVpnToExtrarouteIdentifier(optVpnName.get(), usedRd, vrfEntry.getDestPrefix()));
+                    writeOperTxn.delete(LogicalDatastoreType.CONFIGURATION,
+                            VpnExtraRouteHelper.getUsedRdsIdentifier(vpnId, vrfEntry.getDestPrefix()));
+                }
             }
-            Optional<Adjacencies> optAdjacencies = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                    FibUtil.getAdjListPath(ifName));
+            Optional<Adjacencies> optAdjacencies = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
+                FibUtil.getAdjListPath(ifName));
             int numAdj = 0;
             if (optAdjacencies.isPresent()) {
                 numAdj = optAdjacencies.get().getAdjacency().size();
             }
             //remove adjacency corr to prefix
             if (numAdj > 1) {
-                LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
-                FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
+                LOG.info("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId,
+                        vrfEntry.getDestPrefix());
+                writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL,
                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
-            }
-            if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
+            } else {
+                //this is last adjacency (or) no more adjacency left for this vpn interface, so
                 //clean up the vpn interface from DpnToVpn list
-                LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
-                FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                        FibUtil.getVpnInterfaceIdentifier(ifName));
+                LOG.info("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
+                writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName));
             }
-            return null;
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(writeOperTxn.submit());
+            return futures;
         }
     }
 
     private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
-        final String rd  = vrfTableKey.getRouteDistinguisher();
+        final String rd = vrfTableKey.getRouteDistinguisher();
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
         if (vpnInstance == null) {
             LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
@@ -1268,99 +1451,127 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
         long elanTag = 0L;
         SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+        final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+        List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+        String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
         if (subnetRoute != null) {
             elanTag = subnetRoute.getElantag();
             LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
-                    rd, vrfEntry.getDestPrefix(), elanTag);
+                rd, vrfEntry.getDestPrefix(), elanTag);
             if (vpnToDpnList != null) {
                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
-                        new Callable<List<ListenableFuture<Void>>>() {
-                            @Override
-                            public List<ListenableFuture<Void>> call() throws Exception {
-                                WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-
-                                for (final VpnToDpnList curDpn : vpnToDpnList) {
-
-                                    makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
-                                            vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
-                                    if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
-                                        makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
-                                                DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
-                                    }
-                                }
-                                List<ListenableFuture<Void>> futures = new ArrayList<>();
-                                futures.add(tx.submit());
-                                return futures;
+                dataStoreCoordinator.enqueueJob("FIB-" + rd + "-" + vrfEntry.getDestPrefix(),
+                    () -> {
+                        WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+
+                        for (final VpnToDpnList curDpn : vpnToDpnList) {
+
+                            makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
+                                vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
+                            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                                optionalLabel.ifPresent(label -> {
+                                    makeLFibTableEntry(curDpn.getDpnId(), label, null,
+                                            DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
+                                });
                             }
-                        });
+                        }
+                        List<ListenableFuture<Void>> futures = new ArrayList<>();
+                        futures.add(tx.submit());
+                        return futures;
+                    });
             }
-            synchronized (vrfEntry.getLabel().toString().intern()) {
-                LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-                if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-                    Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd);
-                    String vpnInstanceName = "";
-                    if (vpnInstanceOpDataEntryOptional.isPresent()) {
-                        vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
-                    }
-                    boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
-                    if (lriRemoved) {
-                        String parentRd = lri.getParentVpnRd();
+            optionalLabel.ifPresent(label -> {
+                synchronized (label.toString().intern()) {
+                    LabelRouteInfo lri = getLabelRouteInfo(label);
+                    if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
+                        Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional =
+                                FibUtil.getVpnInstanceOpData(dataBroker, rd);
+                        String vpnInstanceName = "";
+                        if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                            vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                        }
+                        boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
+                        if (lriRemoved) {
+                            String parentRd = lri.getParentVpnRd();
+                            FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
+                                    FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
+                            LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as "
+                                    + "labelRouteInfo cleared", label, rd,
+                                    vrfEntry.getDestPrefix());
+                        }
+                    } else {
                         FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
-                                FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
-                        LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {} as labelRouteInfo cleared", vrfEntry.getLabel(), rd,
-                                vrfEntry.getDestPrefix());
+                                FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
+                        LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}",
+                                label, rd, vrfEntry.getDestPrefix());
                     }
-                } else {
-                    FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
-                            FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
-                    LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
-                            vrfEntry.getDestPrefix());
                 }
-            }
+            });
+            return;
+        }
+        if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) {
             return;
         }
 
         final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
-                vrfTableKey.getRouteDistinguisher(), vrfEntry);
+            vrfTableKey.getRouteDistinguisher(), vrfEntry);
         if (vpnToDpnList != null) {
+            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
+                    vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
+            String usedRd = null;
+            Optional<Routes> extraRouteOptional;
+            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+            if (usedRds != null && !usedRds.isEmpty()) {
+                if (usedRds.size() > 1) {
+                    LOG.error("The extra route prefix is still present in some DPNs");
+                    return ;
+                } else {
+                    // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
+                    //is not present in any other DPN
+                    extraRouteOptional = VpnExtraRouteHelper
+                            .getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
+                }
+            } else {
+                extraRouteOptional = Optional.absent();
+            }
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-
-                            if (localDpnIdList.size() <= 0) {
-                                for (VpnToDpnList curDpn : vpnToDpnList) {
+            dataStoreCoordinator.enqueueJob("FIB-" + usedRd + "-" + vrfEntry.getDestPrefix(),
+                () -> {
+                    WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+
+                    if (localDpnIdList.size() <= 0) {
+                        for (VpnToDpnList curDpn : vpnToDpnList) {
+                            if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
+                                if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
+                                    deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
+                                        vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
+                                }
+                            } else {
+                                deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(),
+                                    vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
+                            }
+                        }
+                    } else {
+                        for (BigInteger localDpnId : localDpnIdList) {
+                            for (VpnToDpnList curDpn : vpnToDpnList) {
+                                if (!curDpn.getDpnId().equals(localDpnId)) {
                                     if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
                                         if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
-                                            deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
+                                            deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
+                                                vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
                                         }
                                     } else {
-                                        deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                    }
-                                }
-                            } else {
-                                for (BigInteger localDpnId : localDpnIdList) {
-                                    for (VpnToDpnList curDpn : vpnToDpnList) {
-                                        if (!curDpn.getDpnId().equals(localDpnId)) {
-                                            if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
-                                                if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
-                                                    deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                                }
-                                            } else {
-                                                deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                            }
-                                        }
+                                        deleteRemoteRoute(localDpnId, curDpn.getDpnId(),
+                                            vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
                                     }
                                 }
                             }
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            futures.add(tx.submit());
-                            return futures;
                         }
-                    });
+                    }
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    futures.add(tx.submit());
+                    return futures;
+                });
         }
 
         //The flow/group entry has been deleted from config DS; need to clean up associated operational
@@ -1370,24 +1581,20 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
         // of the interVpnLink.
         Optional<String> optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd);
-        if ( optVpnUuid.isPresent() ) {
+        if (optVpnUuid.isPresent()) {
             String vpnUuid = optVpnUuid.get();
-            List<String> routeNexthoplist = vrfEntry.getNextHopAddressList();
-            if(routeNexthoplist.isEmpty()) {
-                LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry);
-                return;
-            }
-            String routeNexthop = routeNexthoplist.get(0);
-            Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
-            if ( optInterVpnLink.isPresent() ) {
-                InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
-                if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid))
-                {
-                    // This is route that points to the other endpoint of an InterVpnLink
-                    // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
-                    removeRouteFromInterVpnLink(interVpnLink, rd, vrfEntry);
+            FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
+                Optional<InterVpnLinkDataComposite> optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
+                if (optInterVpnLink.isPresent()) {
+                    InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
+                    if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
+                        // This is route that points to the other endpoint of an InterVpnLink
+                        // In that case, we should look for the FIB table pointing to
+                        // LPortDispatcher table and remove it.
+                        removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
+                    }
                 }
-            }
+            });
         }
 
     }
@@ -1397,28 +1604,45 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
       The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
       provided by ResourceBatchingManager
      */
-    private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
+    private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
+                                  final VrfEntry vrfEntry) {
         final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
-
-        final String rd  = vrfTableKey.getRouteDistinguisher();
+        String rd = vrfTableKey.getRouteDistinguisher();
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
         if (vpnInstance == null) {
             LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
             return;
         }
+        String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
         final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
         if (vpnToDpnList != null) {
+            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
+                    vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
+            Optional<Routes> extraRouteOptional;
+            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+            if (usedRds != null && !usedRds.isEmpty()) {
+                if (usedRds.size() > 1) {
+                    LOG.error("The extra route prefix is still present in some DPNs");
+                    return ;
+                } else {
+                    extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
+                            usedRds.get(0), vrfEntry.getDestPrefix());
+                }
+            } else {
+                extraRouteOptional = Optional.absent();
+            }
             for (VpnToDpnList curDpn : vpnToDpnList) {
                 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
-                    deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, writeTx);
+                    deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey,
+                            vrfEntry, extraRouteOptional, writeTx);
                 }
             }
         }
     }
 
     public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
-                                  final long vpnId, final VrfTablesKey vrfTableKey,
-                                  final VrfEntry vrfEntry, WriteTransaction tx) {
+            final long vpnId, final VrfTablesKey vrfTableKey,
+            final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional, WriteTransaction tx) {
 
         Boolean wrTxPresent = true;
         if (tx == null) {
@@ -1426,35 +1650,33 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             tx = dataBroker.newWriteOnlyTransaction();
         }
 
-        LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
+        LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
+                vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
         String rd = vrfTableKey.getRouteDistinguisher();
 
-        if(localDpnId != null) {
+        if (localDpnId != null && localDpnId != BigInteger.ZERO) {
             // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
+            if (extraRouteOptional.isPresent()) {
+                nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
+            }
             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
             return;
         }
 
         // below two reads are kept as is, until best way is found to identify dpnID
         VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
-        Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-
-        if (localNextHopInfo == null && extraRoute != null) {
-            // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
-            for (String nextHopIp : extraRoute.getNexthopIpList()) {
-                localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
-                checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
-            }
+        if (extraRouteOptional.isPresent()) {
+            nextHopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(), null , false);
         } else {
             checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
         }
-        if(!wrTxPresent ){
+        if (!wrTxPresent) {
             tx.submit();
         }
     }
 
     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
-                                           VrfEntry vrfEntry, String rd, WriteTransaction tx){
+                                           VrfEntry vrfEntry, String rd, WriteTransaction tx) {
         boolean isRemoteRoute = true;
         if (localNextHopInfo != null) {
             isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId());
@@ -1463,23 +1685,33 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
             return true;
         } else {
-            LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
+            LOG.debug("Did not delete FIB entry: rd={}, vrfEntry={}, as it is local to dpnId={}",
+                rd, vrfEntry.getDestPrefix(), remoteDpnId);
             return false;
         }
     }
 
-    private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
+    private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
+            String rd, WriteTransaction tx) {
+        // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
+        // the adjacencyResults.
+        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
+            List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
+            if (!adjacencyResults.isEmpty()) {
+                programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
+                return;
+            }
+        }
         makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
         LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
     }
 
-    private long get
-            (byte[] rawIpAddress) {
+    private long get(byte[] rawIpAddress) {
         return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
-                + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
+            + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
     }
 
-    private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
+    void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
                                     List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
         Boolean wrTxPresent = true;
         if (tx == null) {
@@ -1488,7 +1720,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
 
         LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
-        String values[] = vrfEntry.getDestPrefix().split("/");
+        String[] values = vrfEntry.getDestPrefix().split("/");
         String ipAddress = values[0];
         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
         if (addOrRemove == NwConstants.ADD_FLOW) {
@@ -1506,29 +1738,33 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
         List<MatchInfo> matches = new ArrayList<>();
 
-        matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
-                MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
+        matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_IPV4 }));
-
-        if(prefixLength != 0) {
-            matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
-                    destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
+        if (destPrefix instanceof Inet4Address) {
+            matches.add(MatchEthernetType.IPV4);
+            if (prefixLength != 0) {
+                matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
+            }
+        } else {
+            matches.add(MatchEthernetType.IPV6);
+            if (prefixLength != 0) {
+                matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
+            }
         }
+
         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
-        FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
-                COOKIE_VM_FIB_TABLE, matches, instructions);
-
+        FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority,
+            flowRef, 0, 0,
+            COOKIE_VM_FIB_TABLE, matches, instructions);
         Flow flow = flowEntity.getFlowBuilder().build();
         String flowId = flowEntity.getFlowId();
-        FlowKey flowKey = new FlowKey( new FlowId(flowId));
+        FlowKey flowKey = new FlowKey(new FlowId(flowId));
         Node nodeDpn = buildDpnNode(dpId);
 
         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
+            .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+            .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 
         if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
             SubTransaction subTransaction = new SubTransactionImpl();
@@ -1544,18 +1780,18 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
 
         if (addOrRemove == NwConstants.ADD_FLOW) {
-            tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
+            tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
         } else {
             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
         }
 
-        if(!wrTxPresent ){
+        if (!wrTxPresent) {
             tx.submit();
         }
     }
 
     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
-    private Node buildDpnNode(BigInteger dpnId) {
+    Node buildDpnNode(BigInteger dpnId) {
         NodeId nodeId = new NodeId("openflow:" + dpnId);
         Node nodeDpn = new NodeBuilder().setId(nodeId).setKey(new NodeKey(nodeId)).build();
 
@@ -1570,52 +1806,53 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             tx = dataBroker.newWriteOnlyTransaction();
         }
 
-        List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_MPLS_UC }));
-        matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
+        List<MatchInfo> matches = new ArrayList<>();
+        matches.add(MatchEthernetType.MPLS_UNICAST);
+        matches.add(new MatchMplsLabel(label));
 
         // Install the flow entry in L3_LFIB_TABLE
         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
 
         FlowEntity flowEntity;
         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
-                NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
+            NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
         Flow flow = flowEntity.getFlowBuilder().build();
         String flowId = flowEntity.getFlowId();
-        FlowKey flowKey = new FlowKey( new FlowId(flowId));
+        FlowKey flowKey = new FlowKey(new FlowId(flowId));
         Node nodeDpn = buildDpnNode(dpId);
         InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
+            .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+            .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
 
         if (addOrRemove == NwConstants.ADD_FLOW) {
-            tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
+            tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
         } else {
             tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
         }
-        if(!wrTxPresent ){
+        if (!wrTxPresent) {
             tx.submit();
         }
-        LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
-                dpId, label, instructions );
+
+        LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
+            dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
     }
 
-    private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
-        LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
+    void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
+                                      final String ipPrefixAddress) {
+        LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}", dpId, vpnId, ipAddress);
         try {
-            nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
+            nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress);
         } catch (NullPointerException e) {
             LOG.trace("", e);
         }
     }
 
     public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd,
-            final FutureCallback<List<Void>> callback) {
+                                    final FutureCallback<List<Void>> callback) {
         LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
         if (!vrfTable.isPresent()) {
             LOG.warn("VRF Table not yet available for RD {}", rd);
             if (callback != null) {
@@ -1626,10 +1863,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             return;
         }
         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-        dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" +  dpnId.toString(),
-                new Callable<List<ListenableFuture<Void>>>() {
-            @Override
-            public List<ListenableFuture<Void>> call() throws Exception {
+        dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
+            () -> {
                 List<ListenableFuture<Void>> futures = new ArrayList<>();
                 synchronized (vpnInstance.getVpnInstanceName().intern()) {
                     WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
@@ -1640,19 +1875,37 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                             installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
                             continue;
                         }
-                        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
-                            LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
-                            if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix())
-                                    && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
-                                if (lri.getDpnId().equals(dpnId)) {
-                                    createLocalFibEntry(vpnId, rd, vrfEntry);
-                                    continue;
+                        RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
+                        if (routerInt != null) {
+                            LOG.trace("Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}",
+                                rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
+                            installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
+                                routerInt.getIpAddress(),
+                                new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW);
+                            continue;
+                        }
+                        //Handle local flow creation for imports
+                        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
+                            java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+                            if (optionalLabel.isPresent()) {
+                                List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
+                                LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
+                                if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) {
+                                    if (lri.getDpnId().equals(dpnId)) {
+                                        createLocalFibEntry(vpnId, rd, vrfEntry);
+                                        continue;
+                                    }
                                 }
                             }
                         }
-                        // Passing null as we don't know the dpn
-                        // to which prefix is attached at this point
-                        createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
+
+                        boolean shouldCreateRemoteFibEntry = shouldCreateFibEntryForVrfAndVpnIdOnDpn(vpnId,
+                                vrfEntry, dpnId);
+                        if (shouldCreateRemoteFibEntry) {
+                            LOG.trace("Will create remote FIB entry for vrfEntry {} on DPN {}",
+                                    vrfEntry, dpnId);
+                            createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
+                        }
                     }
                     //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
                     futures.add(tx.submit());
@@ -1662,304 +1915,333 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                     Futures.addCallback(listenableFuture, callback);
                 }
                 return futures;
-            }
-        });
+            });
     }
 
+    public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                            final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
+            dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
+        final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        if (vrfTable.isPresent()) {
+            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+            dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
+                () -> {
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                        WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
+                        vrfTable.get().getVrfEntry().stream()
+                            .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin()))
+                            .forEach(
+                                getConsumerForCreatingRemoteFib(dpnId, vpnId,
+                                        rd, remoteNextHopIp, vrfTable,
+                                        writeCfgTxn));
+                        futures.add(writeCfgTxn.submit());
+                    }
+                    return futures;
+                });
+        }
+    }
 
-    public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
-                                 final String localNextHopIp, final String remoteNextHopIp) {
-        LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
-                dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+    public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                            final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
+            dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
         if (vrfTable.isPresent()) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            synchronized (vpnInstance.getVpnInstanceName().intern()) {
-                                WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
-                                LOG.trace("populate FIB starts on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId );
-                                for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                                    LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
-
-                                    if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
-                                        if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                                            LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
-                                            createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
-                                        }
-                                    } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
-                                        BigInteger dpnIdForPrefix = null;
-                                        String destPfx = vrfEntry.getDestPrefix();
-                                        if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
-                                            Optional<Extraroute> extraRouteInfo =
-                                                    FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                                                            getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
-                                            if (extraRouteInfo.isPresent()) {
-                                                continue;
-                                            }
-                                            dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
-                                        } else {
-                                            // Subnet Route handling
-                                            Optional<Prefixes> localNextHopInfoData =
-                                                    FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                                                            FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
-                                            if (localNextHopInfoData.isPresent()) {
-                                                Prefixes prefixes = localNextHopInfoData.get();
-                                                dpnIdForPrefix = prefixes.getDpnId();
-                                            }
-                                        }
-                                        if (dpnIdForPrefix == null) {
-                                            LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
-                                                    vrfEntry.getDestPrefix());
-                                            continue;
-                                        }
-                                        int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
-                                        if (sameDpnId != 0) {
-                                            LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
-                                                    dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
-                                            continue;
-                                        }
-                                        InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
-                                        List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
-                                        newNextHopAddrList.add(localNextHopIp);
-                                        VrfEntry newVrfEntry =
-                                                new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
-                                        // Just update the VrfEntry
-                                        FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
-                                                vrfEntryId, newVrfEntry);
-                                        // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
-                                        //       vrfEntryId, newVrfEntry);
-                                        vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
-                                        LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
-                                    }
-                                }
-                                futures.add(writeTransaction.submit());
-                                LOG.trace("populate FIB ends on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId);
-                            }
-                            return futures;
-                        }
-                    });
+                () -> {
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                        WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
+                        // Handle Vpn Interface driven Routes only (i.e., STATIC and LOCAL)
+                        vrfTable.get().getVrfEntry().stream()
+                            .filter(vrfEntry -> {
+                                /* Ignore SubnetRoute entry */
+                                return (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
+                                        vrfEntry.getOrigin())));
+                            })
+                            .forEach(getConsumerForCreatingRemoteFib(dpnId, vpnId,
+                                       rd, remoteNextHopIp, vrfTable,
+                                       writeCfgTxn));
+                        futures.add(writeCfgTxn.submit());
+                    }
+                    return futures;
+                });
         }
     }
 
-    public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
-                                  final long vpnId, final String  rd, final String destPrefix ,
-                                  final String localNextHopIP, final String remoteNextHopIp) {
+    public void manageRemoteRouteOnDPN(final boolean action,
+                                       final BigInteger localDpnId,
+                                       final long vpnId,
+                                       final String rd,
+                                       final String destPrefix,
+                                       final String destTepIp,
+                                       final long label) {
+        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 
+        if (vpnInstance == null) {
+            LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
+            return;
+        }
         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-        dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
-                        + "local dpid" + localDpnId
-                        + "remote dpid" + remoteDpnId
-                        + "vpnId" + vpnId
-                        + "localNHIp" + localNextHopIP
-                        + "remoteNHIp" + remoteNextHopIp,
-                new Callable<List<ListenableFuture<Void>>>() {
-                    @Override
-                    public List<ListenableFuture<Void>> call() throws Exception {
-                        List<ListenableFuture<Void>> futures = new ArrayList<>();
-                        WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
-                        VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
-                        VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
-                        if (vrfEntry == null) {
+        dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
+            () -> {
+                List<ListenableFuture<Void>> futures = new ArrayList<>();
+                synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                    WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
+                    VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
+                    VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
+                    if (vrfEntry == null) {
+                        return futures;
+                    }
+                    LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
+                        action, localDpnId, vpnId, rd, destPrefix);
+                    List<RoutePaths> routePathList = vrfEntry.getRoutePaths();
+                    VrfEntry modVrfEntry;
+                    if (routePathList == null || (routePathList.isEmpty())) {
+                        modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label,
+                                Collections.singletonList(destTepIp),
+                                RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build();
+                    } else {
+                        modVrfEntry = vrfEntry;
+                    }
+
+                    if (action == true) {
+                        LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
+                        createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
+                    } else {
+                        LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
+                        List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
+                                vrfEntry.getDestPrefix());
+                        if (usedRds.size() > 1) {
+                            LOG.debug("The extra route prefix is still present in some DPNs");
                             return futures;
                         }
-                        LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
-                                        "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
-                                action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
-                        if (action == true) {
-                            vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
-                            LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
-                            createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
-                        } else {
-                            vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
-                            LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
-                            deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
+                        //Is this fib route an extra route? If yes, get the nexthop which would be
+                        //an adjacency in the vpn
+                        Optional<Routes> extraRouteOptional = Optional.absent();
+                        if (usedRds.size() != 0) {
+                            extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker,
+                                    FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId()),
+                                    usedRds.get(0), vrfEntry.getDestPrefix());
                         }
-                        futures.add(writeTransaction.submit());
-                        return futures;
+                        deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry,
+                                extraRouteOptional, writeTransaction);
                     }
-                });
+                    futures.add(writeTransaction.submit());
+                }
+                return futures;
+            });
     }
 
     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
                                  final FutureCallback<List<Void>> callback) {
-        LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
+        LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
         if (vrfTable.isPresent()) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
             dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            synchronized (vpnInstance.getVpnInstanceName().intern()) {
-                                WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
-                                for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                                    /* Handle subnet routes here */
-                                    SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
-                                    if (subnetRoute != null) {
-                                        LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
-                                                dpnId, rd);
-                                        makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
-                                        makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
-                                        LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
+                () -> {
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                        WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+                        for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                                /* Handle subnet routes here */
+                            SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+                            if (subnetRoute != null) {
+                                LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn",
+                                        vrfEntry.getDestPrefix(),
+                                        dpnId, rd);
+                                makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
+                                java.util.Optional.ofNullable(vrfEntry.getRoutePaths()).ifPresent(routePaths -> {
+                                    routePaths.stream().forEach(routePath -> {
+                                        makeLFibTableEntry(dpnId, routePath.getLabel(), null,
+                                                DEFAULT_FIB_FLOW_PRIORITY,
+                                                NwConstants.DEL_FLOW, tx);
+                                        LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} "
+                                                + "for rd {} prefix {}",
+                                                routePath.getLabel(), rd,
                                                 vrfEntry.getDestPrefix());
-                                        continue;
-                                    }
-                                    // Passing null as we don't know the dpn
-                                    // to which prefix is attached at this point
-                                    deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
-                                }
-                                futures.add(tx.submit());
-                                if (callback != null) {
-                                    ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
-                                    Futures.addCallback(listenableFuture, callback);
+                                    });
+                                });
+                                continue;
+                            }
+                            // ping responder for router interfaces
+                            RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
+                            if (routerInt != null) {
+                                LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}",
+                                    rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
+                                installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(),
+                                    routerInt.getIpAddress(),
+                                    new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
+                                continue;
+                            }
+                            // Passing null as we don't know the dpn
+                            // to which prefix is attached at this point
+                            List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(),
+                                    vrfEntry.getDestPrefix());
+                            String vpnName = FibUtil.getVpnNameFromId(dataBroker, vpnInstance.getVpnId());
+                            Optional<Routes> extraRouteOptional;
+                            //Is this fib route an extra route? If yes, get the nexthop which would be
+                            //an adjacency in the vpn
+                            if (usedRds != null && !usedRds.isEmpty()) {
+                                if (usedRds.size() > 1) {
+                                    LOG.error("The extra route prefix is still present in some DPNs");
+                                    return futures;
+                                } else {
+                                    extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
+                                            usedRds.get(0), vrfEntry.getDestPrefix());
+
                                 }
+                            } else {
+                                extraRouteOptional = Optional.absent();
                             }
-                            return futures;
+                            deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
+                                    extraRouteOptional, tx);
                         }
-
-                    });
+                        futures.add(tx.submit());
+                        if (callback != null) {
+                            ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
+                            Futures.addCallback(listenableFuture, callback);
+                        }
+                    }
+                    return futures;
+                });
         }
     }
 
-    public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
-                                 final String localNextHopIp, final String remoteNextHopIp,
-                                 final FutureCallback<List<Void>> callback) {
-        LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
-                        " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
-                dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+    public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                           final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
+                + " localNexthopIp {} , remoteNexhtHopIp {}",
+            dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
         if (vrfTable.isPresent()) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            synchronized (vpnInstance.getVpnInstanceName().intern()) {
-                                WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
-                                LOG.trace("cleanup FIB starts on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId);
-
-                                for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                                    LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
-                                    if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                                        LOG.trace(" deleting remote FIB entry {}", vrfEntry);
-                                        deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
-                                    }
-
-                                    if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                                        LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
-                                                vrfEntry.getDestPrefix(), dpnId);
-                                        String destPfx = vrfEntry.getDestPrefix();
-                                        InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
-                                        List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
-                                        newList.remove(localNextHopIp);
-                                        VrfEntry newVrfEntry =
-                                                new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
-                                        FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
-                                                vrfEntryId, newVrfEntry);
-                                        vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
-                                        LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
-                                    }
-                                }
-                                futures.add(writeTransaction.submit());
-                                if (callback != null) {
-                                    ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
-                                    Futures.addCallback(listenableFuture, callback);
-                                }
-                                LOG.trace("cleanup FIB ends on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId);
-                            }
-                            return futures;
-                        }
-                    });
+            dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
+                () -> {
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                        WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
+                        vrfTable.get().getVrfEntry().stream()
+                            .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP)
+                            .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId, rd,
+                                remoteNextHopIp, vrfTable, writeCfgTxn));
+                        futures.add(writeCfgTxn.submit());
+                    }
+                    return futures;
+                });
+        }
+    }
 
+    public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                           final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace("cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, "
+                + " localNexthopIp {} , remoteNexhtHopIp {}",
+            dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
+        final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        if (vrfTable.isPresent()) {
+            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+            dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
+                () -> {
+                    List<ListenableFuture<Void>> futures = new ArrayList<>();
+                    synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                        WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
+                        vrfTable.get().getVrfEntry().stream()
+                            .filter(vrfEntry -> {
+                                /* Ignore SubnetRoute entry */
+                                return (FibHelper.isControllerManagedVpnInterfaceRoute(
+                                        RouteOrigin.value(vrfEntry.getOrigin())));
+                            })
+                            .forEach(getConsumerForDeletingRemoteFib(dpnId, vpnId,
+                                rd, remoteNextHopIp, vrfTable, writeCfgTxn));
+                        futures.add(writeCfgTxn.submit());
+                    }
+                    return futures;
+                });
         }
     }
 
     public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
         InstanceIdentifierBuilder<VrfTables> idBuilder =
-                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
+            InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
         InstanceIdentifier<VrfTables> id = idBuilder.build();
         return id;
     }
 
     private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
-        return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
-                .append(priority).toString();
+        return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + label
+                + NwConstants.FLOWID_SEPARATOR + priority;
     }
 
     private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
-        return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(rd).append(NwConstants.FLOWID_SEPARATOR)
-                .append(priority).append(NwConstants.FLOWID_SEPARATOR)
-                .append(destPrefix.getHostAddress()).toString();
+        return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + rd
+                + NwConstants.FLOWID_SEPARATOR + priority + NwConstants.FLOWID_SEPARATOR + destPrefix.getHostAddress();
     }
 
-    private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
-        return new StringBuilder(64).append(FLOWID_PREFIX)
-                .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
-                .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
-                .append(nextHop).toString();
+    private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop) {
+        return FLOWID_PREFIX + interVpnLinkName + NwConstants.FLOWID_SEPARATOR + prefix + NwConstants
+                .FLOWID_SEPARATOR + nextHop;
     }
 
-    protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
-                                            final VrfEntry vrfEntry, String rd) {
-        List<String> adjacencyList = new ArrayList<>();
+    protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
+                                                     final VrfEntry vrfEntry, String rd) {
+        List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
+        FibHelper.sortIpAddress(routePaths);
+        List<AdjacencyResult> adjacencyList = new ArrayList<>();
         List<String> prefixIpList = new ArrayList<>();
-        LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
-                remoteDpnId, vpnId, vrfEntry);
+        LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
+            remoteDpnId, vpnId, vrfEntry);
         try {
             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
-                Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-                if (extra_route == null) {
-                    prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
+                List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix());
+                List<Routes> vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker,
+                        FibUtil.getVpnNameFromId(dataBroker, vpnId), usedRds, vrfEntry.getDestPrefix());
+                if (vpnExtraRoutes.isEmpty()) {
+                    prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
                 } else {
-                    prefixIpList = new ArrayList<>();
-                    for (String extraRouteIp : extra_route.getNexthopIpList()) {
-                        prefixIpList.add(extraRouteIp + "/32");
-                    }
+                    List<String> prefixIpListLocal = new ArrayList<>();
+                    vpnExtraRoutes.stream().forEach(route -> {
+                        route.getNexthopIpList().stream().forEach(extraRouteIp -> {
+                            prefixIpListLocal.add(extraRouteIp + NwConstants.IPV4PREFIX);
+                        });
+                    });
+                    prefixIpList = prefixIpListLocal;
                 }
             } else {
-                prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
+                prefixIpList = Collections.singletonList(vrfEntry.getDestPrefix());
             }
 
             for (String prefixIp : prefixIpList) {
-                for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
-                    LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
-                    String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
-                            prefixIp, nextHopIp);
-                    if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
-                        adjacencyList.add(adjacency);
-                    }
+                if (routePaths == null || routePaths.isEmpty()) {
+                    LOG.trace("Processing Destination IP {} without NextHop IP", prefixIp);
+                    AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
+                          prefixIp, null);
+                    addAdjacencyResultToList(adjacencyList, adjacencyResult);
+                    continue;
                 }
+                adjacencyList.addAll(routePaths.stream()
+                        .map(routePath -> {
+                            LOG.debug("NextHop IP for destination {} is {}", prefixIp,
+                                    routePath.getNexthopAddress());
+                            return nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
+                                    prefixIp, routePath.getNexthopAddress());
+                        })
+                        .filter(adjacencyResult -> adjacencyResult != null && !adjacencyList.contains(adjacencyResult))
+                        .distinct()
+                        .collect(toList()));
             }
         } catch (NullPointerException e) {
             LOG.trace("", e);
@@ -1967,19 +2249,24 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         return adjacencyList;
     }
 
+    private void addAdjacencyResultToList(List<AdjacencyResult> adjacencyList, AdjacencyResult adjacencyResult) {
+        if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
+            adjacencyList.add(adjacencyResult);
+        }
+    }
+
     protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
         InstanceIdentifier<VpnInstanceOpDataEntry> id =
-                InstanceIdentifier.create(VpnInstanceOpData.class)
-                        .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
+            InstanceIdentifier.create(VpnInstanceOpData.class)
+                .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
-                FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
+            MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
         return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
     }
 
     private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) {
-        return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss)
-                .append(FLOWID_PREFIX).toString();
+        return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR
+                + tableMiss + FLOWID_PREFIX;
     }
 
     /*
@@ -1987,21 +2274,20 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
      * coming through gre tunnel to LFIB table.
      */
     private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
-        final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
+        final BigInteger cookieProtocolTable = new BigInteger("1070000", 16);
         // Instruction to goto L3 InterfaceTable
         List<InstructionInfo> instructions = new ArrayList<>();
-        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
-        List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_MPLS_UC }));
+        instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
+        List<MatchInfo> matches = new ArrayList<>();
+        matches.add(MatchEthernetType.MPLS_UNICAST);
         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
-                getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
-                        NwConstants.L3_LFIB_TABLE),
-                DEFAULT_FIB_FLOW_PRIORITY,
-                "Protocol Table For LFIB",
-                0, 0,
-                COOKIE_PROTOCOL_TABLE,
-                matches, instructions);
+            getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
+                NwConstants.L3_LFIB_TABLE),
+            DEFAULT_FIB_FLOW_PRIORITY,
+            "Protocol Table For LFIB",
+            0, 0,
+            cookieProtocolTable,
+            matches, instructions);
 
         if (addOrRemove == NwConstants.ADD_FLOW) {
             LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
@@ -2016,20 +2302,28 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
         result.add("-------------------------------------------------------------------");
         InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
-        Optional<FibEntries> fibEntries = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        Optional<FibEntries> fibEntries = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
         if (fibEntries.isPresent()) {
             List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
             for (VrfTables vrfTable : vrfTables) {
                 for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
-                    for (String nextHop : vrfEntry.getNextHopAddressList()) {
-                        result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
+                    List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
+                    if (routePaths == null) {
+                        result.add(String.format("   %-7s  %-20s  %-20s  %-7s",
                                 vrfTable.getRouteDistinguisher(),
-                                vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
+                                vrfEntry.getDestPrefix(), "local", vrfEntry.getOrigin()));
+                        continue;
                     }
-                    if (vrfEntry.getNextHopAddressList().isEmpty()) {
+                    for (RoutePaths routePath : routePaths) {
                         result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s",
-                                vrfTable.getRouteDistinguisher(),
-                                vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
+                            vrfTable.getRouteDistinguisher(),
+                            vrfEntry.getDestPrefix(), routePath.getNexthopAddress(),
+                            routePath.getLabel(), vrfEntry.getOrigin()));
+                    }
+                    if (routePaths.isEmpty()) {
+                        result.add(String.format("   %-7s  %-20s  %-20s  %-7s",
+                            vrfTable.getRouteDistinguisher(),
+                            vrfEntry.getDestPrefix(), "local", vrfEntry.getOrigin()));
                     }
                 }
             }
@@ -2039,20 +2333,253 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
 
     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
-        InstanceIdentifier<VrfEntry> vrfEntryId =
-                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
-                        child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
-        Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
-        if (vrfEntry.isPresent())  {
+        InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
+            .child(VrfTables.class, new VrfTablesKey(rd))
+            .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
+        Optional<VrfEntry> vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
+        if (vrfEntry.isPresent()) {
             return vrfEntry.get();
         }
         return null;
     }
 
     private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
-        InstanceIdentifier<VrfEntry> vrfEntryId =
-                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
-                        child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
+        InstanceIdentifier<VrfEntry> vrfEntryId = InstanceIdentifier.builder(FibEntries.class)
+            .child(VrfTables.class, new VrfTablesKey(rd))
+            .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
         return vrfEntryId;
     }
+
+    protected Boolean isIpv4Address(String ipAddress) {
+        try {
+            InetAddress address = InetAddress.getByName(ipAddress);
+            if (address instanceof Inet4Address) {
+                return true;
+            }
+        } catch (UnknownHostException e) {
+            LOG.warn("Invalid ip address {}", ipAddress, e);
+            return false;
+        }
+        return false;
+    }
+
+    protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
+                                              long vpnId, int addOrRemove) {
+        RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
+        if (routerInt == null) {
+            return false;
+        }
+        if (vpnToDpnList != null) {
+            String routerId = routerInt.getUuid();
+            String macAddress = routerInt.getMacAddress();
+            String ipValue = routerInt.getIpAddress();
+            LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}",
+                    routerId, ipValue, macAddress);
+            for (VpnToDpnList vpnDpn : vpnToDpnList) {
+                if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
+                    installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue,
+                            new MacAddress(macAddress), addOrRemove);
+                }
+            }
+        }
+        return true;
+    }
+
+    public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
+                                      String routerInternalIp, MacAddress routerMac, int addOrRemove) {
+        String[] subSplit = routerInternalIp.split("/");
+        if (!isIpv4Address(subSplit[0])) {
+            // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
+            return;
+        }
+
+        java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+        if (!optionalLabel.isPresent()) {
+            LOG.warn("Routes paths not present. Exiting installRouterFibEntry");
+            return;
+        }
+        String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
+        LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
+            dpnId, routerInternalIp, vpnId, subSplit[0]);
+
+        List<MatchInfo> matches = new ArrayList<>();
+
+        matches.add(MatchIpProtocol.ICMP);
+        matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
+        matches.add(new MatchIcmpv4((short) 8, (short) 0));
+        matches.add(MatchEthernetType.IPV4);
+        matches.add(new MatchIpv4Destination(subSplit[0], "32"));
+
+        List<ActionInfo> actionsInfos = new ArrayList<>();
+
+        // Set Eth Src and Eth Dst
+        actionsInfos.add(new ActionMoveSourceDestinationEth());
+        actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
+
+        // Move Ip Src to Ip Dst
+        actionsInfos.add(new ActionMoveSourceDestinationIp());
+        actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
+
+        // Set the ICMP type to 0 (echo reply)
+        actionsInfos.add(new ActionSetIcmpType((short) 0));
+
+        actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
+
+        actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
+
+        List<InstructionInfo> instructions = new ArrayList<>();
+
+        instructions.add(new InstructionApplyActions(actionsInfos));
+
+        int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
+        String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, optionalLabel.get(),
+                priority);
+
+        FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef,
+            0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions);
+
+        if (addOrRemove == NwConstants.ADD_FLOW) {
+            mdsalManager.installFlow(flowEntity);
+        } else {
+            mdsalManager.removeFlow(flowEntity);
+        }
+    }
+
+    public void removeInterVPNLinkRouteFlows(final InterVpnLinkDataComposite interVpnLink,
+                                             final String vpnName,
+                                             final VrfEntry vrfEntry) {
+        Preconditions.checkArgument(vrfEntry.getRoutePaths() != null && vrfEntry.getRoutePaths().size() == 1);
+
+        String interVpnLinkName = interVpnLink.getInterVpnLinkName();
+        List<BigInteger> targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName);
+
+        if (targetDpns.isEmpty()) {
+            LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName);
+            return;
+        }
+
+        java.util.Optional<String> optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry);
+        java.util.Optional<Long> optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
+
+        // delete from FIB
+        //
+        optNextHop.ifPresent(nextHop -> {
+            String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
+            FlowKey flowKey = new FlowKey(new FlowId(flowRef));
+            Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
+                    .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef).build();
+
+            LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", interVpnLinkName, flowRef);
+            for (BigInteger dpId : targetDpns) {
+                LOG.debug("Removing flow: VrfEntry=[prefix={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
+                          vrfEntry.getDestPrefix(), nextHop, dpId, interVpnLinkName);
+
+                mdsalManager.removeFlow(dpId, flow);
+            }
+        });
+
+        // delete from LFIB
+        //
+        optLabel.ifPresent(label -> {
+            LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
+
+            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+            for (BigInteger dpId : targetDpns) {
+                LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB",
+                          vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName);
+                makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
+            }
+            tx.submit();
+        });
+    }
+
+
+    private Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
+            final BigInteger dpnId, final long vpnId, final String rd,
+            final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
+            WriteTransaction writeCfgTxn) {
+        return vrfEntry -> vrfEntry.getRoutePaths().stream()
+                .filter(routes -> !routes.getNexthopAddress().isEmpty()
+                        && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
+                .findFirst()
+                .ifPresent(routes -> {
+                    LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
+                            vrfEntry.getDestPrefix(), rd, dpnId);
+                    createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
+                });
+    }
+
+    private Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
+            final BigInteger dpnId, final long vpnId, final String rd,
+            final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
+            WriteTransaction writeCfgTxn) {
+        return vrfEntry -> vrfEntry.getRoutePaths().stream()
+                .filter(routes -> !routes.getNexthopAddress().isEmpty()
+                        && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
+                .findFirst()
+                .ifPresent(routes -> {
+                    LOG.trace(" deleting remote FIB entry {}", vrfEntry);
+                    deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry,
+                            Optional.absent(), writeCfgTxn);
+                });
+    }
+
+    private boolean isPrefixAndNextHopPresentInLri(String prefix,
+            List<String> nextHopAddressList, LabelRouteInfo lri) {
+        return lri != null && lri.getPrefix().equals(prefix)
+                && nextHopAddressList.contains(lri.getNextHopIpList().get(0));
+    }
+
+    private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) {
+        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
+            return true;
+        }
+
+        Prefixes prefix = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
+        if (prefix != null) {
+            BigInteger prefixDpnId = prefix.getDpnId();
+            if (prefixDpnId == dpnId) {
+                LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}",
+                        vrfEntry, dpnId);
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId, final long vpnId,
+            final VrfEntry vrfEntry, WriteTransaction tx, String rd, List<AdjacencyResult> adjacencyResults) {
+        Preconditions.checkArgument(vrfEntry.getRoutePaths().size() <= 2);
+
+        if (adjacencyResults.size() == 1) {
+            programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults);
+            return;
+        }
+        // ECMP Use case, point to LB group. Move the mpls label accordingly.
+        List<String> tunnelList =
+                adjacencyResults.stream()
+                .map(adjacencyResult -> adjacencyResult.getNextHopIp())
+                .sorted().collect(toList());
+        String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
+        long groupId = nextHopManager.createNextHopPointer(lbGroupKey);
+        int index = 0;
+        List<ActionInfo> actionInfos = new ArrayList<>();
+        for (AdjacencyResult adjResult : adjacencyResults) {
+            String nextHopIp = adjResult.getNextHopIp();
+            java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
+            if (!optionalLabel.isPresent()) {
+                LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
+                continue;
+            }
+            long label = optionalLabel.get();
+
+            actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
+                    31, label));
+        }
+        List<InstructionInfo> instructions = new ArrayList<>();
+        actionInfos.add(new ActionGroup(index, groupId));
+        instructions.add(new InstructionApplyActions(actionInfos));
+        makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+    }
 }