MatchInfo redesign
[netvirt.git] / vpnservice / fibmanager / fibmanager-impl / src / main / java / org / opendaylight / netvirt / fibmanager / VrfEntryListener.java
old mode 100644 (file)
new mode 100755 (executable)
index 7227c63..8d76ff9
@@ -12,37 +12,58 @@ import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
-
 import java.math.BigInteger;
+import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
-
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
-import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
+import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
 import org.opendaylight.genius.mdsalutil.ActionInfo;
-import org.opendaylight.genius.mdsalutil.ActionType;
 import org.opendaylight.genius.mdsalutil.FlowEntity;
 import org.opendaylight.genius.mdsalutil.InstructionInfo;
-import org.opendaylight.genius.mdsalutil.InstructionType;
 import org.opendaylight.genius.mdsalutil.MDSALUtil;
-import org.opendaylight.genius.mdsalutil.MatchFieldType;
 import org.opendaylight.genius.mdsalutil.MatchInfo;
 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
+import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
+import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
+import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
+import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
+import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
+import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetIcmpType;
+import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
+import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
+import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
+import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
+import org.opendaylight.genius.mdsalutil.matches.MatchIcmpv4;
+import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
+import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
+import org.opendaylight.genius.mdsalutil.matches.MatchIpv6Destination;
+import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
+import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
+import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
 import org.opendaylight.genius.mdsalutil.packet.IPProtocols;
 import org.opendaylight.genius.utils.ServiceIndex;
 import org.opendaylight.genius.utils.batching.ActionableResource;
@@ -51,11 +72,13 @@ import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
 import org.opendaylight.genius.utils.batching.ResourceHandler;
 import org.opendaylight.genius.utils.batching.SubTransaction;
 import org.opendaylight.genius.utils.batching.SubTransactionImpl;
+import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
@@ -92,14 +115,10 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
@@ -154,6 +173,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         this.nextHopManager = nexthopManager;
         this.interfaceManager = interfaceManager;
         this.idManager = idManager;
+
         batchSize = Integer.getInteger("batch.size");
         if (batchSize == null) {
             batchSize = BATCH_SIZE;
@@ -199,9 +219,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             actResource.setInstanceIdentifier(identifier);
             actResource.setInstance(vrfEntry);
             vrfEntryBufferQ.add(actResource);
-            leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
         }
-        LOG.debug("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
+        leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
+        LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
     }
 
@@ -219,29 +239,71 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             actResource.setInstanceIdentifier(identifier);
             actResource.setInstance(vrfEntry);
             vrfEntryBufferQ.add(actResource);
-            leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
         }
-        LOG.debug("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
+        leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
+        LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
                 rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
     }
 
     @Override
     protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
         Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
-        String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
+
+        final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
+        final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
         LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
-        if (RouteOrigin.value(update.getOrigin()) != RouteOrigin.BGP) {
-            createFibEntries(identifier, update);
-        } else {
+        // Handle BGP Routes first
+        if (RouteOrigin.value(update.getOrigin()) == RouteOrigin.BGP) {
             ActionableResource actResource = new ActionableResourceImpl(rd.toString() + update.getDestPrefix());
             actResource.setAction(ActionableResource.UPDATE);
             actResource.setInstanceIdentifier(identifier);
             actResource.setInstance(update);
             actResource.setOldInstance(original);
             vrfEntryBufferQ.add(actResource);
+            LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
+                    rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
+            return;
+        }
+
+        // Handle Internal Routes next (ie., STATIC only)
+          if (FibUtil.isControllerManagedNonInterVpnLinkRoute(RouteOrigin.value(update.getOrigin()))) {
+            SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
+            /* Ignore SubnetRoute entry, as it will be driven by createFibEntries call down below */
+            if (subnetRoute == null) {
+                List<String> origNhList = original.getNextHopAddressList();
+                List<String> updateNhList = update.getNextHopAddressList();
+                //final SubnetRoute subnetRoute = update.getAugmentation(SubnetRoute.class);
+                LOG.info("UPDATE: Original nexthop {} updateNextHop {} ", origNhList, updateNhList);
+
+                // If original VRF Entry had nexthop null , but update VRF Entry
+                // has nexthop , route needs to be created on remote Dpns
+                if (((origNhList == null) || (origNhList.isEmpty()) &&
+                        (updateNhList != null) && (!updateNhList.isEmpty()))) {
+                    // TODO(vivek): Though ugly, Not handling this code now, as each
+                    // tep add event will invoke flow addition
+                    LOG.trace("Original VRF entry NH is null for destprefix {}. This event is IGNORED here.", update.getDestPrefix());
+                    return;
+                }
+
+                // If original VRF Entry had valid nexthop , but update VRF Entry
+                // has nexthop empty'ed out, route needs to be removed from remote Dpns
+                if (((updateNhList == null) || (updateNhList.isEmpty()) &&
+                        (origNhList != null) && (!origNhList.isEmpty()))) {
+                    LOG.trace("Original VRF entry had valid NH for destprefix {}. This event is IGNORED here.", update.getDestPrefix());
+                    return;
+                }
+            }
+            createFibEntries(identifier, update);
+            LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
+                    rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
+            return;
         }
-        LOG.debug("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
+
+        /* Handl all other route origins */
+        createFibEntries(identifier, update);
+
+        LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
                 rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
     }
 
@@ -302,7 +364,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                     rd, vrfEntry.getDestPrefix(), elanTag);
             if (vpnToDpnList != null) {
                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
+                dataStoreCoordinator.enqueueJob("FIB-"+ rd.toString() + "-" + vrfEntry.getDestPrefix(),
                         new Callable<List<ListenableFuture<Void>>>() {
                             @Override
                             public List<ListenableFuture<Void>> call() throws Exception {
@@ -325,17 +387,11 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             return;
         }
 
-        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
-            // When it is a leaked route, the LFIB and FIB goes a bit different.
-            installInterVpnRouteInLFib(rd, vrfEntry);
-            return;
-        }
-
         final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
 
         if (vpnToDpnList != null) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
+            dataStoreCoordinator.enqueueJob("FIB-"+ rd.toString() + "-" + vrfEntry.getDestPrefix(),
                     new Callable<List<ListenableFuture<Void>>>() {
                         @Override
                         public List<ListenableFuture<Void>> call() throws Exception {
@@ -365,7 +421,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid) ) {
                     // This is an static route that points to the other endpoint of an InterVpnLink
                     // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
-                    installRouteInInterVpnLink(interVpnLink, vpnUuid, vrfEntry, vpnId);
+                    installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
                     installInterVpnRouteInLFib(rd, vrfEntry);
                 }
             }
@@ -396,6 +452,12 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
     }
 
+    private boolean originMustBeLeaked(InterVpnLink ivpnLink, RouteOrigin routeOrigin) {
+        return    routeOrigin == RouteOrigin.BGP && ivpnLink.isBgpRoutesLeaking()
+               || routeOrigin == RouteOrigin.STATIC && ivpnLink.isStaticRoutesLeaking()
+               || routeOrigin == RouteOrigin.CONNECTED && ivpnLink.isConnectedRoutesLeaking();
+    }
+
     // FIXME: Refactoring needed here.
     //        This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager
     private void leakRouteIfNeeded(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry,
@@ -405,14 +467,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
         String rd = vrfTableKey.getRouteDistinguisher();
         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
-            if (vpnInstance == null) {
-                LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
-                return;
-            }
-        } else {
-            Preconditions.checkNotNull(vpnInstance,
-                                       "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
+        if (vpnInstance == null) {
+            LOG.error("VPN Instance not available for route with prefix {} label {} nextHop {} RD {}. Returning...",
+                      vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
+            return;
         }
         String vpnUuid = vpnInstance.getVpnInstanceName();
         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
@@ -434,16 +492,17 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
         boolean proceed =
-            (addOrRemove == NwConstants.DEL_FLOW) || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
-                                                       && interVpnLink.get().isBgpRoutesLeaking() );
+            addOrRemove == NwConstants.DEL_FLOW || originMustBeLeaked(interVpnLink.get(),
+                                                                      RouteOrigin.value(vrfEntry.getOrigin()));
 
         if ( proceed ) {
-            String theOtherVpnId = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid)
-                    ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
-                    : vpnUuid;
+            boolean isVpnFirstEndpoint = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid);
 
+            String theOtherVpnId = isVpnFirstEndpoint ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
+                                                      : interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue();
             String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId);
-            String endpointIp = vrfEntry.getNextHopAddressList().get(0);
+            String endpointIp = isVpnFirstEndpoint ? interVpnLink.get().getFirstEndpoint().getIpAddress().getValue()
+                                                   : interVpnLink.get().getSecondEndpoint().getIpAddress().getValue();
 
             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
                     InstanceIdentifier.builder(FibEntries.class)
@@ -516,8 +575,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId).shiftLeft(1)));
-        instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
-        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
+        instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
+        instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 
         if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
@@ -525,10 +584,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             // reinitialize instructions list for LFIB Table
             final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 
-            actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
-            LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
-            LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
-            LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
+            actionsInfos.add(new ActionPopMpls());
+            LFIBinstructions.add(new InstructionApplyActions(actionsInfos));
+            LFIBinstructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE));
+            LFIBinstructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE));
 
             makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
         }
@@ -537,6 +596,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
     }
 
+    /*
+     * For a given route, it installs a flow in LFIB that sets the lportTag of the other endpoint and sends to
+     * LportDispatcher table (via table 80)
+     */
     private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) {
         // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing
         // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the
@@ -571,18 +634,23 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag()
                                                  : vpnLinkState.get().getFirstEndpointState().getLportTag();
 
+                LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName());
+
                 for ( BigInteger dpId : targetDpns ) {
-                    List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
+                    List<ActionInfo> actionsInfos = Collections.singletonList(new ActionPopMpls());
 
-                    BigInteger[] metadata = new BigInteger[] {
-                            MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
-                            MetaDataUtil.getMetaDataMaskForLPortDispatcher()
-                    };
                     List<InstructionInfo> instructions =
-                            Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
-                                    new InstructionInfo(InstructionType.write_metadata, metadata),
-                                    new InstructionInfo(InstructionType.goto_table,
-                                            new long[] { NwConstants.L3_INTERFACE_TABLE }));
+                            Arrays.asList(new InstructionApplyActions(actionsInfos),
+                                    new InstructionWriteMetadata(
+                                            MetaDataUtil.getMetaDataForLPortDispatcher(lportTag.intValue(),
+                                                    ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME,
+                                                            NwConstants.L3VPN_SERVICE_INDEX)),
+                                            MetaDataUtil.getMetaDataMaskForLPortDispatcher()),
+                                    new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE));
+
+                    LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
+                              vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
+                              dpId, interVpnLink.getName());
 
                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
                             NwConstants.ADD_FLOW, null);
@@ -599,9 +667,11 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
     }
 
 
-
-    private void installRouteInInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
-                                            final VrfEntry vrfEntry, long vpnTag) {
+    /*
+     * Installs the flows in FIB table that, for a given route, do the switching from one VPN to the other.
+     */
+    private void installIVpnLinkSwitchingFlows(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
+                                               final VrfEntry vrfEntry, long vpnTag) {
         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
                 && vrfEntry.getNextHopAddressList().size() == 1);
@@ -631,28 +701,24 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             return;
         }
 
-        BigInteger[] metadata = new BigInteger[] {
-                MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
-                        ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)),
-                MetaDataUtil.getMetaDataMaskForLPortDispatcher()
-        };
         List<Instruction> instructions =
-                Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
-                              new InstructionInfo(InstructionType.goto_table,
-                                                  new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
+                Arrays.asList(new InstructionWriteMetadata(
+                                MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(),
+                                        ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants
+                                                .L3VPN_SERVICE_INDEX)),
+                                MetaDataUtil.getMetaDataMaskForLPortDispatcher()).buildInstruction(0),
+                              new InstructionGotoTable(NwConstants.L3_INTERFACE_TABLE).buildInstruction(1));
 
         String values[] = destination.split("/");
         String destPrefixIpAddress = values[0];
         int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
 
         List<MatchInfo> matches = new ArrayList<>();
-        matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag),
-                                                                              MetaDataUtil.METADATA_MASK_VRFID }));
-        matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
+        matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID));
+        matches.add(MatchEthernetType.IPV4);
 
         if (prefixLength != 0) {
-            matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
-                                      new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
+            matches.add(new MatchIpv4Destination(destPrefixIpAddress, Integer.toString(prefixLength)));
         }
 
         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
@@ -660,38 +726,20 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
                                                  COOKIE_VM_FIB_TABLE, matches, instructions);
 
-        for ( BigInteger dpId : targetDpns ) {
-            mdsalManager.installFlow(dpId, flowEntity);
-        }
-    }
-
-    private void removeRouteFromInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid,
-                                             final VrfEntry vrfEntry) {
-
-        Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
-        Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
-                                    && vrfEntry.getNextHopAddressList().size() == 1);
-
-        String iVpnLinkName = interVpnLink.getInterVpnLinkName();
+        LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}" ,
+                  vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef);
 
-        InterVpnLinkState interVpnLinkState = interVpnLink.getInterVpnLinkState();
-        if ( interVpnLinkState == null ) {
-            LOG.warn("Could not find State for InterVpnLink {}", iVpnLinkName);
-            return;
-        }
+        for ( BigInteger dpId : targetDpns ) {
 
-        String nextHop = vrfEntry.getNextHopAddressList().get(0);
-        String flowRef = getInterVpnFibFlowRef(iVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
-        FlowId flowId = new FlowId(flowRef);
-        Flow flow = new FlowBuilder().setKey(new FlowKey(flowId)).setId(flowId).setTableId(NwConstants.L3_FIB_TABLE)
-                                     .setFlowName(flowRef).build();
+            LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB",
+                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(),
+                     dpId, interVpnLink.getInterVpnLinkName());
 
-        for ( BigInteger dpId : interVpnLink.getEndpointDpnsByVpnName(vpnUuid) ) {
-            mdsalManager.removeFlow(dpId, flow);
+            mdsalManager.installFlow(dpId, flowEntity);
         }
-
     }
 
+
     private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
                                                      InstanceIdentifier<T> path) {
 
@@ -707,9 +755,34 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         return result;
     }
 
+    private List<BigInteger> getDpnIdForPrefix(DataBroker broker, Long vpnId, String rd, VrfEntry vrfEntry) {
+        List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
+        Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, vrfEntry.getDestPrefix());
+
+        if (localNextHopInfo == null) {
+            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+            Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+            if (extraRoute != null) {
+                for (String nextHopIp : extraRoute.getNexthopIpList()) {
+                    LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
+                    if (nextHopIp != null) {
+                        localNextHopInfo = FibUtil.getPrefixToInterface(broker, vpnId, nextHopIp + "/32");
+                        if (localNextHopInfo != null) {
+                            returnLocalDpnId.add(localNextHopInfo.getDpnId());
+                        }
+                    }
+                }
+            }
+        } else {
+            returnLocalDpnId.add(localNextHopInfo.getDpnId());
+        }
+
+        return returnLocalDpnId;
+    }
+
     private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
         List<BigInteger> returnLocalDpnId = new ArrayList<>();
-        Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
+        Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
         String localNextHopIP = vrfEntry.getDestPrefix();
 
         if (localNextHopInfo == null) {
@@ -719,7 +792,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 for (String nextHopIp : extraRoute.getNexthopIpList()) {
                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
                     if (nextHopIp != null) {
-                        localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
+                        localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
                         localNextHopIP = nextHopIp + "/32";
                         BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
                         returnLocalDpnId.add(dpnId);
@@ -778,13 +851,12 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                         vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString());
                 return BigInteger.ZERO;
             }
-            List<ActionInfo> actionsInfos =
-                    Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
-            final List<InstructionInfo> instructions =
-                    Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
-            actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
-                    new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
-            final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+            final List<InstructionInfo> instructions = Collections.singletonList(
+                new InstructionApplyActions(
+                    Collections.singletonList(new ActionGroup(groupId))));
+            final List<InstructionInfo> lfibinstructions = Collections.singletonList(
+                new InstructionApplyActions(
+                    Arrays.asList(new ActionPopMpls(), new ActionGroup(groupId))));
             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
                 LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
                         dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
@@ -792,7 +864,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
             }
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
+            dataStoreCoordinator.enqueueJob("FIB-"+ vpnId.toString() + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
                     new Callable<List<ListenableFuture<Void>>>() {
                         @Override
                         public List<ListenableFuture<Void>> call() throws Exception {
@@ -857,9 +929,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
                                       WriteTransaction tx) {
-        List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
-        actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
-
+        List<ActionInfo> actionsInfos = Collections.singletonList(new ActionGroup(groupId));
 
         createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 
@@ -875,10 +945,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
         // Matching metadata
         // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
-        mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+        mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
 
         List<InstructionInfo> mkInstructions = new ArrayList<>();
-        mkInstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+        mkInstructions.add(new InstructionApplyActions(actionsInfos));
 
         FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
                 getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
@@ -900,7 +970,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
         List<MatchInfo> mkMatches = new ArrayList<>();
         // Matching metadata
-        mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+        mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label)));
         flowEntity = MDSALUtil.buildFlowEntity(dpId,
                 NwConstants.INTERNAL_TUNNEL_TABLE,
                 getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
@@ -979,7 +1049,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         if (localNextHopInfo != null) {
             final BigInteger dpnId = localNextHopInfo.getDpnId();;
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
+            dataStoreCoordinator.enqueueJob("FIB-"+ vpnId.toString() + "-" + dpnId.toString() + "-" + vrfEntry.getDestPrefix(),
                     new Callable<List<ListenableFuture<Void>>>() {
                         @Override
                         public List<ListenableFuture<Void>> call() throws Exception {
@@ -1003,18 +1073,6 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         return BigInteger.ZERO;
     }
 
-
-    private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
-        return InstanceIdentifier.builder(PrefixToInterface.class)
-                .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
-    }
-
-    private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
-        Optional<Prefixes> localNextHopInfoData =
-                FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
-        return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
-    }
-
     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
         return InstanceIdentifier.builder(VpnToExtraroute.class)
                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
@@ -1054,12 +1112,12 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             tx = dataBroker.newWriteOnlyTransaction();
         }
         String rd = vrfTableKey.getRouteDistinguisher();
-        LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
-                vrfEntry.getDestPrefix(), rd, tx);
+        LOG.debug(  "createremotefibentry: adding route {} for rd {} on remoteDpnId {}",
+                vrfEntry.getDestPrefix(), rd, remoteDpnId);
         /********************************************/
-        List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
+        List<AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
 
-        if (tunnelInterfaceList.isEmpty()) {
+        if (adjacencyResults.isEmpty()) {
             LOG.error("Could not get interface for nexthop: {} in vpn {}",
                     vrfEntry.getNextHopAddressList(), rd);
             LOG.warn("Failed to add Route: {} in vpn: {}",
@@ -1067,34 +1125,24 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             return;
         }
 
-        for (String tunnelInterface : tunnelInterfaceList) {
+        for (AdjacencyResult adjacencyResult : adjacencyResults) {
             List<InstructionInfo> instructions = new ArrayList<>();
             List<ActionInfo> actionInfos = new ArrayList<>();
-            Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
-            if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
-                LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
-                actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null}));
-                actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())}));
+            String egressInterface = adjacencyResult.getInterfaceName();
+            if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) {
+                addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos);
             } else {
-                int label = vrfEntry.getLabel().intValue();
-                BigInteger tunnelId;
-                // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
-                if (tunnel_type.equals(TunnelTypeVxlan.class)) {
-                    tunnelId = BigInteger.valueOf(label);
-                } else {
-                    tunnelId = BigInteger.valueOf(label);
-                }
-
-                LOG.debug("adding set tunnel id action for label {}", label);
-                actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId}));
+                addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
             }
-            List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface);
-            if(egressActions.isEmpty()){
-                LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface);
+            List<ActionInfo> egressActions = nextHopManager.getEgressActionsForInterface(egressInterface);
+            if (egressActions.isEmpty()) {
+                LOG.error(
+                        "Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.",
+                        vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface);
                 return;
             }
             actionInfos.addAll(egressActions);
-            instructions.add(new InstructionInfo(InstructionType.apply_actions, actionInfos));
+            instructions.add(new InstructionApplyActions(actionInfos));
             makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
         }
         if(!wrTxPresent ){
@@ -1103,6 +1151,54 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
     }
 
+    private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List<ActionInfo> actionInfos) {
+        String ipPrefix = vrfEntry.getDestPrefix();
+        Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix);
+        if (prefixInfo == null) {
+            LOG.debug("No prefix info found for prefix {}", ipPrefix);
+            return;
+        }
+
+        String ifName = prefixInfo.getVpnInterfaceName();
+        if (ifName == null) {
+            LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix);
+            return;
+        }
+
+        String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix);
+        if (macAddress == null) {
+            LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix);
+            return;
+        }
+
+        actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(), new MacAddress(macAddress)));
+    }
+
+    private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry,
+            List<ActionInfo> actionInfos) {
+        Class<? extends TunnelTypeBase> tunnelType = getTunnelType(tunnelInterface);
+        if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
+            LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
+            actionInfos.add(new ActionPushMpls());
+            actionInfos.add(new ActionSetFieldMplsLabel(vrfEntry.getLabel()));
+            actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
+        } else {
+            int label = vrfEntry.getLabel().intValue();
+            BigInteger tunnelId;
+            // FIXME vxlan vni bit set is not working properly with OVS.need to
+            // revisit
+            if (tunnelType.equals(TunnelTypeVxlan.class)) {
+                tunnelId = BigInteger.valueOf(label);
+            } else {
+                tunnelId = BigInteger.valueOf(label);
+            }
+
+            LOG.debug("adding set tunnel id action for label {}", label);
+            actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
+            addRewriteDstMacAction(vpnId, vrfEntry, actionInfos);
+        }
+    }
+
     private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
@@ -1135,7 +1231,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
          - vpn interface op DS
      */
         LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
-        Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
+        Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix());
         Extraroute extraRoute = null;
         if (prefixInfo == null) {
             extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
@@ -1144,7 +1240,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                     LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
 
                     if (nextHopIp != null) {
-                        prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
+                        prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32");
                         checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
                     }
                 }
@@ -1282,7 +1378,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                     rd, vrfEntry.getDestPrefix(), elanTag);
             if (vpnToDpnList != null) {
                 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
+                dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
                         new Callable<List<ListenableFuture<Void>>>() {
                             @Override
                             public List<ListenableFuture<Void>> call() throws Exception {
@@ -1336,7 +1432,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 vrfTableKey.getRouteDistinguisher(), vrfEntry);
         if (vpnToDpnList != null) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
+            dataStoreCoordinator.enqueueJob("FIB-" + rd.toString() + "-" + vrfEntry.getDestPrefix(),
                     new Callable<List<ListenableFuture<Void>>>() {
                         @Override
                         public List<ListenableFuture<Void>> call() throws Exception {
@@ -1396,7 +1492,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 {
                     // This is route that points to the other endpoint of an InterVpnLink
                     // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
-                    removeRouteFromInterVpnLink(interVpnLink, rd, vrfEntry);
+                    removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(),
+                                                 interVpnLink.isFirstEndpointVpnName(rd),
+                                                 vrfEntry);
                 }
             }
         }
@@ -1437,7 +1535,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             tx = dataBroker.newWriteOnlyTransaction();
         }
 
-        LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
+        LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}", vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
         String rd = vrfTableKey.getRouteDistinguisher();
 
         if(localDpnId != null) {
@@ -1517,16 +1615,20 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
         List<MatchInfo> matches = new ArrayList<>();
 
-        matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
-                MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
-
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_IPV4 }));
+        matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 
-        if(prefixLength != 0) {
-            matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
-                    destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
+        if (destPrefix instanceof Inet4Address) {
+            matches.add(MatchEthernetType.IPV4);
+            if(prefixLength != 0) {
+                matches.add(new MatchIpv4Destination(destPrefix.getHostAddress(), Integer.toString(prefixLength)));
+            }
+        } else {
+            matches.add(MatchEthernetType.IPV6);
+            if(prefixLength != 0) {
+                matches.add(new MatchIpv6Destination(destPrefix.getHostAddress() + "/" + prefixLength));
+            }
         }
+
         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
         String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
@@ -1582,9 +1684,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
 
         List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_MPLS_UC }));
-        matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
+        matches.add(MatchEthernetType.MPLS_UNICAST);
+        matches.add(new MatchMplsLabel(label));
 
         // Install the flow entry in L3_LFIB_TABLE
         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority);
@@ -1608,8 +1709,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         if(!wrTxPresent ){
             tx.submit();
         }
-        LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",
-                dpId, label, instructions );
+
+        LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully",
+                  dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED");
     }
 
     private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress,
@@ -1687,9 +1789,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
     }
 
 
-    public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
-                                 final String localNextHopIp, final String remoteNextHopIp) {
-        LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
+    public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                            final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
@@ -1702,72 +1804,57 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                         public List<ListenableFuture<Void>> call() throws Exception {
                             List<ListenableFuture<Void>> futures = new ArrayList<>();
                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
-                                WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
-                                LOG.trace("populate FIB starts on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId );
+                                WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                                    LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
-
-                                    if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
-                                        if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                                            LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
-                                            createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
-                                        }
-                                    } else if (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue())) {
-                                        BigInteger dpnIdForPrefix = null;
-                                        String destPfx = vrfEntry.getDestPrefix();
-                                        if (vrfEntry.getAugmentation(SubnetRoute.class) == null) {
-                                            Optional<Extraroute> extraRouteInfo =
-                                                    FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                                                            getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
-                                            if (extraRouteInfo.isPresent()) {
-                                                continue;
+                                    if(!vrfEntry.getNextHopAddressList().isEmpty()) {
+                                        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
+                                            if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                                LOG.trace(" creating remote FIB entry for prefix {} rd {}", vrfEntry.getDestPrefix(), rd);
+                                                createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
                                             }
-                                            dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
-                                        } else {
-                                            // Subnet Route handling
-                                            Optional<Prefixes> localNextHopInfoData =
-                                                    FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
-                                                            FibUtil.getPrefixToInterfaceIdentifier(vpnId, destPfx));
-                                            if (localNextHopInfoData.isPresent()) {
-                                                Prefixes prefixes = localNextHopInfoData.get();
-                                                dpnIdForPrefix = prefixes.getDpnId();
-                                            }
-                                        }
-                                        if (dpnIdForPrefix == null) {
-                                            LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
-                                                    vrfEntry.getDestPrefix());
-                                            continue;
                                         }
-                                        int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
-                                        if (sameDpnId != 0) {
-                                            LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
-                                                    dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
-                                            continue;
+                                    }
+                                }
+                                futures.add(writeCfgTxn.submit());
+                            }
+                            return futures;
+                        }
+                    });
+        }
+    }
+
+    public void populateInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                            final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace("populateInternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ",
+                dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
+        final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        if (vrfTable.isPresent()) {
+            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+            dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
+                    new Callable<List<ListenableFuture<Void>>>() {
+                        @Override
+                        public List<ListenableFuture<Void>> call() throws Exception {
+                            List<ListenableFuture<Void>> futures = new ArrayList<>();
+                            synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                                WriteTransaction writeCfgTxn = dataBroker.newWriteOnlyTransaction();
+                                for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                                    // Handle Internal Routes only (i.e., STATIC for now)
+                                    if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
+                                        SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+                                        /* Ignore SubnetRoute entry */
+                                        if (subnetRoute == null) {
+                                            if(!vrfEntry.getNextHopAddressList().isEmpty()) {
+                                                if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                                    LOG.trace(" creating remote FIB entry for prefix {} rd {} on Dpn {}", vrfEntry.getDestPrefix(), rd, dpnId);
+                                                    createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeCfgTxn);
+                                                }
+                                            }
                                         }
-                                        InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
-                                        List<String> newNextHopAddrList = vrfEntry.getNextHopAddressList();
-                                        newNextHopAddrList.add(localNextHopIp);
-                                        VrfEntry newVrfEntry =
-                                                new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newNextHopAddrList).build();
-                                        // Just update the VrfEntry
-                                        FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
-                                                vrfEntryId, newVrfEntry);
-                                        // writeTransaction.put(LogicalDatastoreType.CONFIGURATION,
-                                        //       vrfEntryId, newVrfEntry);
-                                        vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
-                                        LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
                                     }
                                 }
-                                futures.add(writeTransaction.submit());
-                                LOG.trace("populate FIB ends on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId);
+                                futures.add(writeCfgTxn.submit());
                             }
                             return futures;
                         }
@@ -1775,40 +1862,51 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         }
     }
 
-    public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId,
-                                  final long vpnId, final String  rd, final String destPrefix ,
-                                  final String localNextHopIP, final String remoteNextHopIp) {
+    public void manageRemoteRouteOnDPN(final boolean action,
+                                       final BigInteger localDpnId,
+                                       final long vpnId,
+                                       final String  rd,
+                                       final String destPrefix,
+                                       final String destTepIp) {
+        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
 
+        if (vpnInstance == null) {
+            LOG.error("VpnInstance for rd {} not present for prefix {}", rd, destPrefix);
+            return;
+        }
         DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-        dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
-                        + "local dpid" + localDpnId
-                        + "remote dpid" + remoteDpnId
-                        + "vpnId" + vpnId
-                        + "localNHIp" + localNextHopIP
-                        + "remoteNHIp" + remoteNextHopIp,
+        dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + localDpnId.toString(),
                 new Callable<List<ListenableFuture<Void>>>() {
                     @Override
                     public List<ListenableFuture<Void>> call() throws Exception {
                         List<ListenableFuture<Void>> futures = new ArrayList<>();
-                        WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
-                        VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
-                        VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
-                        if (vrfEntry == null) {
-                            return futures;
-                        }
-                        LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
-                                        "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
-                                action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
-                        if (action == true) {
-                            vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
-                            LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
-                            createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
-                        } else {
-                            vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
-                            LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
-                            deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
+                        synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                            WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
+                            VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
+                            VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix);
+                            if (vrfEntry == null)
+                                return futures;
+                            LOG.trace("manageRemoteRouteOnDPN :: action {}, DpnId {}, vpnId {}, rd {}, destPfx {}",
+                                    action, localDpnId, vpnId, rd, destPrefix);
+                            List<String> nhList = new ArrayList<String>();
+                            List<String> nextHopAddressList = vrfEntry.getNextHopAddressList();
+                            VrfEntry modVrfEntry;
+                            if (nextHopAddressList == null || (nextHopAddressList.isEmpty())) {
+                                nhList = Arrays.asList(destTepIp);
+                                modVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(nhList).build();
+                            } else {
+                                modVrfEntry = vrfEntry;
+                            }
+
+                            if (action == true) {
+                                LOG.trace("manageRemoteRouteOnDPN updated(add)  vrfEntry :: {}", modVrfEntry);
+                                createRemoteFibEntry(localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
+                            } else {
+                                LOG.trace("manageRemoteRouteOnDPN updated(remove)  vrfEntry :: {}", modVrfEntry);
+                                deleteRemoteRoute(null, localDpnId, vpnId, vrfTablesKey, modVrfEntry, writeTransaction);
+                            }
+                            futures.add(writeTransaction.submit());
                         }
-                        futures.add(writeTransaction.submit());
                         return futures;
                     }
                 });
@@ -1816,7 +1914,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
     public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
                                  final FutureCallback<List<Void>> callback) {
-        LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
+        LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
@@ -1862,72 +1960,82 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                             }
                             return futures;
                         }
-
                     });
         }
     }
 
-    public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
-                                 final String localNextHopIp, final String remoteNextHopIp,
-                                 final FutureCallback<List<Void>> callback) {
-        LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
-                        " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
+    public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                           final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace(  "cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, " +
+                        " localNexthopIp {} , remoteNexhtHopIp {}",
                 dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
         InstanceIdentifier<VrfTables> id = buildVrfId(rd);
         final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
         final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
         if (vrfTable.isPresent()) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob(" FIB-" + vpnId + "-" + dpnId.toString(),
+            dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
                     new Callable<List<ListenableFuture<Void>>>() {
                         @Override
                         public List<ListenableFuture<Void>> call() throws Exception {
                             List<ListenableFuture<Void>> futures = new ArrayList<>();
                             synchronized (vpnInstance.getVpnInstanceName().intern()) {
                                 WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
-                                LOG.trace("cleanup FIB starts on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId);
-
                                 for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                                    LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
-                                    if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                                        LOG.trace(" deleting remote FIB entry {}", vrfEntry);
-                                        deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
+                                    if(!vrfEntry.getNextHopAddressList().isEmpty()) {
+                                        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
+                                            if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                                LOG.trace(" deleting remote FIB entry {}", vrfEntry);
+                                                deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
+                                            }
+                                        }
                                     }
+                                }
+                                futures.add(writeTransaction.submit());
+                            }
+                            return futures;
+                        }
+                    });
 
-                                    if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                                        LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
-                                                vrfEntry.getDestPrefix(), dpnId);
-                                        String destPfx = vrfEntry.getDestPrefix();
-                                        InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
-                                        List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
-                                        newList.remove(localNextHopIp);
-                                        VrfEntry newVrfEntry =
-                                                new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
-                                        FibUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION,
-                                                vrfEntryId, newVrfEntry);
-                                        vrfEntry = getVrfEntry(dataBroker, rd, destPfx);
-                                        LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
+        }
+    }
+    public void cleanUpInternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd,
+                                           final String localNextHopIp, final String remoteNextHopIp) {
+        LOG.trace(  "cleanUpInternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, " +
+                        " localNexthopIp {} , remoteNexhtHopIp {}",
+                dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
+        final Optional<VrfTables> vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        if (vrfTable.isPresent()) {
+            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+            dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(),
+                    new Callable<List<ListenableFuture<Void>>>() {
+                        @Override
+                        public List<ListenableFuture<Void>> call() throws Exception {
+                            List<ListenableFuture<Void>> futures = new ArrayList<>();
+                            synchronized (vpnInstance.getVpnInstanceName().intern()) {
+                                WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction();
+                                for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                                    // Handle Internal Routes only (i.e, STATIC for now)
+                                    if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC) {
+                                        SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+                                        /* Ignore SubnetRoute entry */
+                                        if (subnetRoute == null) {
+                                            if (!vrfEntry.getNextHopAddressList().isEmpty()) {
+                                                if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                                    LOG.trace(" deleting remote FIB entry {}", vrfEntry);
+                                                    deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
+                                                }
+                                            }
+                                        }
                                     }
                                 }
                                 futures.add(writeTransaction.submit());
-                                if (callback != null) {
-                                    ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
-                                    Futures.addCallback(listenableFuture, callback);
-                                }
-                                LOG.trace("cleanup FIB ends on Dpn " + dpnId
-                                        + "rd  " + rd.toString()
-                                        + "localNextHopIp " + localNextHopIp
-                                        + "remoteNextHopIp" + remoteNextHopIp
-                                        + "vpnId " + vpnId);
                             }
                             return futures;
                         }
                     });
-
         }
     }
 
@@ -1959,11 +2067,11 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                 .append(nextHop).toString();
     }
 
-    protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
+    protected List<AdjacencyResult> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId,
                                             final VrfEntry vrfEntry, String rd) {
-        List<String> adjacencyList = new ArrayList<>();
+        List<AdjacencyResult> adjacencyList = new ArrayList<>();
         List<String> prefixIpList = new ArrayList<>();
-        LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}",
+        LOG.trace("resolveAdjacency called with remotedDpnId {}, vpnId{}, VrfEntry {}",
                 remoteDpnId, vpnId, vrfEntry);
         try {
             if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.BGP) {
@@ -1983,10 +2091,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             for (String prefixIp : prefixIpList) {
                 for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
                     LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
-                    String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
+                    AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId,
                             prefixIp, nextHopIp);
-                    if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
-                        adjacencyList.add(adjacency);
+                    if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) {
+                        adjacencyList.add(adjacencyResult);
                     }
                 }
             }
@@ -2019,10 +2127,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
         // Instruction to goto L3 InterfaceTable
         List<InstructionInfo> instructions = new ArrayList<>();
-        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
+        instructions.add(new InstructionGotoTable(NwConstants.L3_LFIB_TABLE));
         List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_MPLS_UC }));
+        matches.add(MatchEthernetType.MPLS_UNICAST);
         FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
                 getTableMissFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE,
                         NwConstants.L3_LFIB_TABLE),
@@ -2085,6 +2192,19 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         return vrfEntryId;
     }
 
+    protected Boolean isIpv4Address(String ipAddress) {
+        try {
+            InetAddress address = InetAddress.getByName(ipAddress);
+            if (address instanceof Inet4Address) {
+                return true;
+            }
+        } catch (UnknownHostException e) {
+            LOG.warn("Invalid ip address {}", ipAddress, e);
+            return false;
+        }
+        return false;
+    }
+
     protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection<VpnToDpnList> vpnToDpnList,
             long vpnId, int addOrRemove) {
         RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
@@ -2108,6 +2228,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
     public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid,
                                       String routerInternalIp, MacAddress routerMac, int addOrRemove) {
         String[] subSplit = routerInternalIp.split("/");
+        if (!isIpv4Address(subSplit[0])) {
+            // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping.
+            return;
+        }
 
         String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW";
         LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr,
@@ -2115,34 +2239,32 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
 
         List<MatchInfo> matches = new ArrayList<>();
 
-        matches.add(new MatchInfo(MatchFieldType.ip_proto, new long[] { IPProtocols.ICMP.intValue() }));
-        matches.add(new MatchInfo(MatchFieldType.metadata,
-                new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
-        matches.add(new MatchInfo(MatchFieldType.icmp_v4, new long[] { (short) 8, (short) 0 }));
-        matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
-        matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { subSplit[0], "32" }));
+        matches.add(MatchIpProtocol.ICMP);
+        matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
+        matches.add(new MatchIcmpv4((short) 8, (short) 0));
+        matches.add(MatchEthernetType.IPV4);
+        matches.add(new MatchIpv4Destination(subSplit[0], "32"));
 
         List<ActionInfo> actionsInfos = new ArrayList<>();
 
         // Set Eth Src and Eth Dst
-        actionsInfos.add(new ActionInfo(ActionType.move_src_dst_eth, new String[] {}));
-        actionsInfos.add(new ActionInfo(ActionType.set_field_eth_src, new String[] { routerMac.getValue() }));
+        actionsInfos.add(new ActionMoveSourceDestinationEth());
+        actionsInfos.add(new ActionSetFieldEthernetSource(routerMac));
 
         // Move Ip Src to Ip Dst
-        actionsInfos.add(new ActionInfo(ActionType.move_src_dst_ip, new String[] {}));
-        actionsInfos.add(new ActionInfo(ActionType.set_source_ip, new String[] { subSplit[0], "32" }));
+        actionsInfos.add(new ActionMoveSourceDestinationIp());
+        actionsInfos.add(new ActionSetSourceIp(subSplit[0], "32"));
 
         // Set the ICMP type to 0 (echo reply)
-        actionsInfos.add(new ActionInfo(ActionType.set_icmp_type, new String[] { "0" }));
+        actionsInfos.add(new ActionSetIcmpType((short) 0));
 
-        actionsInfos.add(new ActionInfo(ActionType.nx_load_in_port, new BigInteger[]{ BigInteger.ZERO }));
+        actionsInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
 
-        actionsInfos.add(new ActionInfo(ActionType.nx_resubmit,
-                new String[] { Short.toString(NwConstants.L3_FIB_TABLE) }));
+        actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 
         List<InstructionInfo> instructions = new ArrayList<>();
 
-        instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+        instructions.add(new InstructionApplyActions(actionsInfos));
 
         int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH;
         String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority);
@@ -2156,4 +2278,56 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
             mdsalManager.removeFlow(flowEntity);
         }
     }
+
+    public void removeInterVPNLinkRouteFlows(final String interVpnLinkName,
+                                             final boolean isVpnFirstEndPoint,
+                                             final VrfEntry vrfEntry)
+    {
+        Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
+                                    && vrfEntry.getNextHopAddressList().size() == 1);
+        Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName);
+
+        if ( !interVpnLinkState.isPresent()) {
+            LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName);
+            return;
+        }
+
+        List<BigInteger> targetDpns =
+            isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
+            : interVpnLinkState.get().getSecondEndpointState().getDpId();
+
+        String nextHop = vrfEntry.getNextHopAddressList().get(0);
+
+        // delete from FIB
+        //
+        String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop);
+        FlowKey flowKey = new FlowKey(new FlowId(flowRef));
+        Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
+            .setFlowName(flowRef).build();
+
+        LOG.trace("Removing flow in FIB table for interVpnLink {} key {}",
+                  interVpnLinkName, flowRef);
+
+        for ( BigInteger dpId : targetDpns ) {
+            LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB",
+                      vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
+                      dpId, interVpnLinkName);
+
+            mdsalManager.removeFlow(dpId, flow);
+        }
+
+        // delete from LFIB
+        //
+        LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName);
+
+        WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+        for ( BigInteger dpId : targetDpns ) {
+            LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB",
+                      vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop,
+                      dpId, interVpnLinkName);
+            makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */,
+                               LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx);
+        }
+        tx.submit();
+    }
 }