Bug 5781: Ping test is FAILED from DC-GW to Invisible IP
[vpnservice.git] / fibmanager / fibmanager-impl / src / main / java / org / opendaylight / vpnservice / fibmanager / FibManager.java
index d3d584201d7d97fb3026d06c495143fbd5b03883..769038f1cae35ebb43367547565317cc1b4d18b7 100644 (file)
@@ -17,6 +17,7 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -43,6 +44,10 @@ import org.opendaylight.vpnservice.mdsalutil.NwConstants;
 import org.opendaylight.vpnservice.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.PrefixToInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnInstanceOpData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.RdToElanOp;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.rd.to.elan.op.RdToElanOpEntry;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.rd.to.elan.op.RdToElanOpEntryKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.VpnToExtraroute;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.prefix.to._interface.VpnIds;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
@@ -51,8 +56,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instanc
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.GetTunnelInterfaceNameOutput;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.itm.rpcs.rev151217.ItmRpcService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.to.extraroute.Vpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.FibEntries;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTables;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.fibmanager.rev150330.fibentries.VrfTablesKey;
@@ -65,6 +75,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpc
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.GetTunnelTypeOutput;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.interfacemgr.rpcs.rev151003.OdlInterfaceRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.Adjacencies;
 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 import org.opendaylight.yangtools.yang.binding.DataObject;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
@@ -83,30 +94,15 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   private NexthopManager nextHopManager;
   private ItmRpcService itmManager;
   private OdlInterfaceRpcService interfaceManager;
-
-  private static final short L3_FIB_TABLE = 21;
-  private static final short L3_LFIB_TABLE = 20;
-  private static final short L3_PROTOCOL_TABLE = 36;
-  private static final short L3_INTERFACE_TABLE = 80;
-  public static final short LPORT_DISPATCHER_TABLE = 30;
+  private IdManagerService idManager;
   private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000002", 16);
   private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
   private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
   private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
   private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
+  public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 
 
-  private static final FutureCallback<Void> DEFAULT_CALLBACK =
-      new FutureCallback<Void>() {
-        public void onSuccess(Void result) {
-          LOG.debug("Success in Datastore write operation");
-        }
-
-        public void onFailure(Throwable error) {
-          LOG.error("Error in Datastore write operation", error);
-        };
-      };
-
   public FibManager(final DataBroker db) {
     super(VrfEntry.class);
     broker = db;
@@ -146,9 +142,13 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
       this.interfaceManager = ifManager;
   }
 
+  public void setIdManager(IdManagerService idManager) {
+      this.idManager = idManager;
+  }
+
   private void registerListener(final DataBroker db) {
     try {
-      listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
+      listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
                                                            getWildCardPath(), FibManager.this, DataChangeScope.SUBTREE);
     } catch (final Exception e) {
       LOG.error("FibManager DataChange listener registration fail!", e);
@@ -156,48 +156,28 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     }
   }
 
-  private <T extends DataObject> Optional<T> read(LogicalDatastoreType datastoreType,
-                                                  InstanceIdentifier<T> path) {
-
-    ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
-
-    Optional<T> result = Optional.absent();
-    try {
-      result = tx.read(datastoreType, path).get();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-
-    return result;
-  }
 
   private InstanceIdentifier<VrfEntry> getWildCardPath() {
     return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
   }
 
-  private <T extends DataObject> void asyncWrite(LogicalDatastoreType datastoreType,
-                                                 InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
-    WriteTransaction tx = broker.newWriteOnlyTransaction();
-    tx.put(datastoreType, path, data, true);
-    Futures.addCallback(tx.submit(), callback);
-  }
 
   @Override
   protected void add(final InstanceIdentifier<VrfEntry> identifier,
                      final VrfEntry vrfEntry) {
-    LOG.trace("key: " + identifier + ", value=" + vrfEntry );
+    LOG.trace("Add key: " + identifier + ", value=" + vrfEntry );
     createFibEntries(identifier, vrfEntry);
   }
 
   @Override
   protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
-    LOG.trace("key: " + identifier + ", value=" + vrfEntry);
+    LOG.trace("Remove key: " + identifier + ", value=" + vrfEntry);
     deleteFibEntries(identifier, vrfEntry);
   }
 
   @Override
   protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
-    LOG.trace("key: " + identifier + ", original=" + original + ", update=" + update );
+    LOG.trace("Update key: " + identifier + ", original=" + original + ", update=" + update );
     createFibEntries(identifier, update);
   }
 
@@ -208,47 +188,147 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 
     VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
-    Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available!");
-    Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + "has null vpnId!");
+    Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
+    Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
 
     Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
+    Long vpnId = vpnInstance.getVpnId();
+    String rd = vrfTableKey.getRouteDistinguisher();
+    RdToElanOpEntry rdToElanOpEntry = getRdToElanOpEntry(broker, rd,
+            vrfEntry.getDestPrefix());
+    if (rdToElanOpEntry!=null) {
+        if (vpnToDpnList!=null) {
+            for (VpnToDpnList curDpn :  vpnToDpnList) {
+                installSubnetRouteInFib(curDpn.getDpnId(), rdToElanOpEntry, vpnId.longValue(), vrfEntry);
+            }
+        }
+        return;
+    }
+    BigInteger localDpnId = createLocalFibEntry(vpnInstance.getVpnId(),
+            rd, vrfEntry);
     if (vpnToDpnList != null) {
-      BigInteger localDpnId = createLocalFibEntry(vpnInstance.getVpnId(),
-                          vrfTableKey.getRouteDistinguisher(), vrfEntry);
-      for (VpnToDpnList curDpn : vpnToDpnList) {
-        if (!curDpn.getDpnId().equals(localDpnId)) {
-          createRemoteFibEntry(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(),
-                               vrfTableKey, vrfEntry);
+        for (VpnToDpnList curDpn : vpnToDpnList) {
+            if (!curDpn.getDpnId().equals(localDpnId)) {
+                createRemoteFibEntry(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(),
+                        vrfTableKey, vrfEntry);
+            }
         }
-      }
     }
   }
 
+  private void installSubnetRouteInFib(BigInteger dpnId, RdToElanOpEntry rdToElanOpEntry,
+                                       long vpnId, VrfEntry vrfEntry){
+      makeSubnetRouteFlow(dpnId);
+      List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
+      Long elanTag = rdToElanOpEntry.getElanTag();
+
+      instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { (BigInteger.valueOf(elanTag)).shiftLeft(24), MetaDataUtil.METADATA_MASK_SERVICE }));
+      instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
+      makeConnectedRoute(dpnId,vpnId,vrfEntry,rdToElanOpEntry.getRd(),
+              instructions,NwConstants.ADD_FLOW);
+
+      List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
+      // reinitialize instructions list for LFIB Table
+      instructions = new ArrayList<InstructionInfo>();
+
+      actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
+      instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+      instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { (BigInteger.valueOf(elanTag)).shiftLeft(24), MetaDataUtil.METADATA_MASK_SERVICE }));
+      instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
+
+      makeLFibTableEntry(dpnId,vrfEntry.getLabel(),instructions,
+              vrfEntry.getNextHopAddress(),NwConstants.ADD_FLOW);
+      // TODO makeTunnelTableEntry();
+  }
+
+  private RdToElanOpEntry getRdToElanOpEntry(DataBroker broker, String rd, String subnetIp) {
+      InstanceIdentifier<RdToElanOpEntry> id = getRdToElanOpEntryDataPath(rd,subnetIp);
+      Optional<RdToElanOpEntry> sn = read(broker, LogicalDatastoreType.OPERATIONAL, id);
+      if(sn.isPresent()) {
+          return sn.get();
+      }
+      return null;
+  }
+
+  private InstanceIdentifier<RdToElanOpEntry> getRdToElanOpEntryDataPath(String rd, String subnetIp) {
+      return InstanceIdentifier.builder(RdToElanOp.class).child(RdToElanOpEntry.class,
+              new RdToElanOpEntryKey(rd,subnetIp)).build();
+  }
+  private  <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                  InstanceIdentifier<T> path) {
+
+      ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
+
+      Optional<T> result = Optional.absent();
+      try {
+          result = tx.read(datastoreType, path).get();
+      } catch (Exception e) {
+          throw new RuntimeException(e);
+      }
+
+      return result;
+  }
+
+  private void makeSubnetRouteFlow(BigInteger dpnId) {
+      //Ask Vivek cookie
+      final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16);
+      List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
+      List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
+      actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[]{}));
+      instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
+      List<MatchInfo> matches = new ArrayList<MatchInfo>();
+      String flowRef = getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, NwConstants.TABLE_MISS_FLOW);
+      FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef,
+              NwConstants.TABLE_MISS_PRIORITY, "Subnet Route Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
+
+      LOG.debug("Invoking MDSAL to install Table Miss Entries");
+      mdsalManager.syncInstallFlow(flowEntity,1);
+  }
+
+  private Collection<BigInteger> getDpnsForVpn(VpnInstanceOpDataEntry vpnInstance) {
+      Collection<BigInteger> dpns = new HashSet<>();
+      for(VpnToDpnList dpn : vpnInstance.getVpnToDpnList()) {
+          dpns.add(dpn.getDpnId());
+      }
+
+      return dpns;
+  }
+
   public BigInteger createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
     BigInteger localDpnId = BigInteger.ZERO;
     Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
-    boolean staticRoute = false;
+    String localNextHopIP = vrfEntry.getDestPrefix();
 
-    //If the vrf entry is a static/extra route, the nexthop of the entry would be a adjacency in the vpn
     if(localNextHopInfo == null) {
-      localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getNextHopAddress() + "/32");
-      staticRoute = true;
+        //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+        Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+        if (extra_route != null) {
+            localNextHopInfo = getPrefixToInterface(vpnId, extra_route.getNexthopIp() + "/32");
+            localNextHopIP = extra_route.getNexthopIp() + "/32";
+        }
     }
 
     if(localNextHopInfo != null) {
-      localDpnId = localNextHopInfo.getDpnId();
-      long groupId = nextHopManager.createLocalNextHop(vpnId, localDpnId, localNextHopInfo.getVpnInterfaceName(),
-                                                        (staticRoute == true) ? vrfEntry.getNextHopAddress() + "/32" : vrfEntry.getDestPrefix());
-      List<ActionInfo> actionInfos = new ArrayList<ActionInfo>();
+        localDpnId = localNextHopInfo.getDpnId();
+        long groupId = nextHopManager.createLocalNextHop(vpnId, localDpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
 
-      actionInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
+        List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
+        List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
+
+        actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
+        instructions.add(new InstructionInfo(InstructionType.write_actions,actionsInfos));
+        makeConnectedRoute(localDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW);
 
-      makeConnectedRoute(localDpnId, vpnId, vrfEntry, rd, actionInfos, NwConstants.ADD_FLOW);
-      makeLFibTableEntry(localDpnId, vrfEntry.getLabel(), groupId, vrfEntry.getNextHopAddress(), NwConstants.ADD_FLOW);
+        actionsInfos=new ArrayList<ActionInfo>();;
+        instructions=new ArrayList<InstructionInfo>();
+        actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
+        actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
+        instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
+        makeLFibTableEntry(localDpnId, vrfEntry.getLabel(), instructions, vrfEntry.getNextHopAddress(), NwConstants.ADD_FLOW);
 
-      LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}", 
-                      localDpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
-      makeTunnelTableEntry(localDpnId, vrfEntry.getLabel(), groupId);
+        LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
+                localDpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
+        makeTunnelTableEntry(localDpnId, vrfEntry.getLabel(), groupId);
 
     }
     return localDpnId;
@@ -266,54 +346,65 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   }
 
   public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos) {
-    // FIXME
-/*      List<MatchInfo> mkMatches = new ArrayList<MatchInfo>();
+      List<MatchInfo> mkMatches = new ArrayList<MatchInfo>();
 
       LOG.info("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 
       // Matching metadata
-      mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {
-                                  MetaDataUtil.getTunnelIdWithValidVniBitAndVniSet(label),
-                                  MetaDataUtil.METADA_MASK_TUNNEL_ID }));
+      // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
+      mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 
       List<InstructionInfo> mkInstructions = new ArrayList<InstructionInfo>();
       mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 
-      FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId,ITMConstants.TERMINATING_SERVICE_TABLE,
-                      getFlowRef(destDpId, ITMConstants.TERMINATING_SERVICE_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
-                      0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
+      FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
+                      getFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
+                      0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 
-      mdsalManager.installFlow(terminatingServiceTableFlowEntity);*/
+      mdsalManager.installFlow(terminatingServiceTableFlowEntity);
  }
 
   private void removeTunnelTableEntry(BigInteger dpId, long label) {
-      // FIXME
-      // itmManager.removeTerminatingServiceAction(dpId, (int)label);
-
-      // LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully {}",dpId, label);
+    FlowEntity flowEntity;
+    LOG.info("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
+    List<MatchInfo> mkMatches = new ArrayList<MatchInfo>();
+    // Matching metadata
+    mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+    flowEntity = MDSALUtil.buildFlowEntity(dpId,
+                                           NwConstants.INTERNAL_TUNNEL_TABLE,
+                                           getFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
+                                           5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
+                                           COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
+    mdsalManager.removeFlow(flowEntity);
+    LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully {}",dpId, label);
   }
 
   public BigInteger deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
     BigInteger localDpnId = BigInteger.ZERO;
+    boolean isExtraRoute = false;
     VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
-    boolean staticRoute = false;
+    String localNextHopIP = vrfEntry.getDestPrefix();
 
-    //If the vrf entry is a static/extra route, the nexthop of the entry would be a adjacency in the vpn
     if(localNextHopInfo == null) {
-      localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getNextHopAddress() + "/32");
-      staticRoute = true;
+        //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+        Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+        if (extra_route != null) {
+            localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, extra_route.getNexthopIp() + "/32");
+            localNextHopIP = extra_route.getNexthopIp() + "/32";
+            isExtraRoute = true;
+        }
     }
 
+
     if(localNextHopInfo != null) {
       localDpnId = localNextHopInfo.getDpnId();
-      if (getPrefixToInterface(vpnId, (staticRoute == true) ? vrfEntry.getNextHopAddress() + "/32" : vrfEntry.getDestPrefix()) == null) {
+      Prefixes prefix = getPrefixToInterface(vpnId, isExtraRoute ? localNextHopIP : vrfEntry.getDestPrefix());
         makeConnectedRoute(localDpnId, vpnId, vrfEntry, rd, null /* invalid */,
                            NwConstants.DEL_FLOW);
-        makeLFibTableEntry(localDpnId, vrfEntry.getLabel(), 0 /* invalid */,
+        makeLFibTableEntry(localDpnId, vrfEntry.getLabel(), null /* invalid */,
                            vrfEntry.getNextHopAddress(), NwConstants.DEL_FLOW);
         removeTunnelTableEntry(localDpnId, vrfEntry.getLabel());
-        deleteLocalAdjacency(localDpnId, vpnId, (staticRoute == true) ? vrfEntry.getNextHopAddress() + "/32" : vrfEntry.getDestPrefix());
-      }
+        deleteLocalAdjacency(localDpnId, vpnId, localNextHopIP);
     }
     return localDpnId;
   }
@@ -325,10 +416,22 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
   private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
     Optional<Prefixes> localNextHopInfoData =
-        read(LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
+        FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
     return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
   }
-  
+
+    private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
+        return InstanceIdentifier.builder(VpnToExtraroute.class)
+                .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
+                        new ExtrarouteKey(ipPrefix)).build();
+    }
+
+    private Extraroute getVpnToExtraroute(String rd, String ipPrefix) {
+        Optional<Extraroute> extraRouteInfo =
+                FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(rd, ipPrefix));
+        return  extraRouteInfo.isPresent() ? extraRouteInfo.get() : null;
+
+    }
 
   private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
         try {
@@ -354,7 +457,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     String rd = vrfTableKey.getRouteDistinguisher();
     LOG.debug("adding route " + vrfEntry.getDestPrefix() + " " + rd);
     /********************************************/
-    String tunnelInterface = resolveAdjacency(localDpnId, remoteDpnId, vpnId, vrfEntry);
+    String tunnelInterface = resolveAdjacency(localDpnId, remoteDpnId, vpnId, vrfEntry, rd);
     if(tunnelInterface == null) {
       LOG.error("Could not get interface for nexthop: {} in vpn {}",
                                    vrfEntry.getNextHopAddress(), rd);
@@ -362,7 +465,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                              vrfEntry.getDestPrefix(), rd);
       return;
     }
-        List<ActionInfo> actionInfos = nextHopManager.getEgressActionsForInterface(tunnelInterface);
+      List<ActionInfo> actionInfos = new ArrayList<>();
        Class<? extends TunnelTypeBase> tunnel_type = getTunnelType(tunnelInterface);
     if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) {
         LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix());
@@ -371,15 +474,19 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     } else {
         int label = vrfEntry.getLabel().intValue();
         BigInteger tunnelId;
+        // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
         if(tunnel_type.equals(TunnelTypeVxlan.class)) {
-               tunnelId = MetaDataUtil.getTunnelIdWithValidVniBitAndVniSet(label);
+               tunnelId = BigInteger.valueOf(label);
         } else {
                tunnelId = BigInteger.valueOf(label);
         }
         LOG.debug("adding set tunnel id action for label {}", label);
-        actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[] {
-                       tunnelId, MetaDataUtil.METADA_MASK_VALID_TUNNEL_ID_BIT_AND_TUNNEL_ID }));
+        actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{
+                tunnelId}));
     }
+    actionInfos.addAll(nextHopManager.getEgressActionsForInterface(tunnelInterface));
+    List<InstructionInfo> instructions= new ArrayList<InstructionInfo>();
+    instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
 /*
     List<ActionInfo> actionInfos = resolveAdjacency(localDpnId, remoteDpnId, vpnId, vrfEntry);
     if(actionInfos == null) {
@@ -407,9 +514,81 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                 MetaDataUtil.METADA_MASK_VALID_TUNNEL_ID_BIT_AND_TUNNEL_ID }));
     }
 **/
-    makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, actionInfos, NwConstants.ADD_FLOW);
-    LOG.debug(
-        "Successfully added fib entry for " + vrfEntry.getDestPrefix() + " vpnId " + vpnId);
+      makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW);
+      LOG.debug("Successfully added fib entry for " + vrfEntry.getDestPrefix() + " vpnId " + vpnId);
+  }
+
+  private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) {
+      InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
+      Optional<VpnToDpnList> dpnInVpn = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
+      if (dpnInVpn.isPresent()) {
+          List<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
+                  .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
+          org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
+                  currVpnInterface = new org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder().setInterfaceName(intfName).build();
+
+          if (vpnInterfaces.remove(currVpnInterface)) {
+              if (vpnInterfaces.isEmpty()) {
+                  LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
+                  FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id);
+                  cleanUpDpnForVpn(dpnId, vpnId, rd);
+              } else {
+                  LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
+                  FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
+                          org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data
+                                  .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
+                          new org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey(intfName)));
+              }
+          }
+      }
+  }
+
+  private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
+    /* Get interface info from prefix to interface mapping;
+        Use the interface info to get the corresponding vpn interface op DS entry,
+        remove the adjacency corresponding to this fib entry.
+        If adjacency removed is the last adjacency, clean up the following:
+         - vpn interface from dpntovpn list, dpn if last vpn interface on dpn
+         - prefix to interface entry
+         - vpn interface op DS
+     */
+      Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
+      Extraroute extraRoute = null;
+      if (prefixInfo == null) {
+          extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+          if(extraRoute != null) {
+              prefixInfo = getPrefixToInterface(vpnId, extraRoute.getNexthopIp() + "/32");
+              //clean up the vpn to extra route entry in DS
+              FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
+          }
+      }
+      if (prefixInfo == null) {
+          LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for " + vrfEntry.getDestPrefix());
+          return; //Don't have any info for this prefix (shouldn't happen); need to return
+      }
+      String ifName = prefixInfo.getVpnInterfaceName();
+      Optional<Adjacencies> optAdjacencies = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, FibUtil.getAdjListPath(ifName));
+      int numAdj = 0;
+      if (optAdjacencies.isPresent()) {
+          numAdj = optAdjacencies.get().getAdjacency().size();
+      }
+      LOG.trace("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
+      //remove adjacency corr to prefix
+      if (numAdj > 1) {
+        FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
+                         FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
+      }
+
+      if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
+          //clean up the vpn interface from DpnToVpn list
+          LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
+          FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
+                         FibUtil.getVpnInterfaceIdentifier(ifName));
+      }
+
+      FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
+         FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
+
   }
 
   private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier,
@@ -418,19 +597,45 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     Preconditions.checkNotNull(vrfTableKey, "VrfTablesKey cannot be null or empty!");
     Preconditions.checkNotNull(vrfEntry, "VrfEntry cannot be null or empty!");
 
+    String rd  = vrfTableKey.getRouteDistinguisher();
     VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
-    Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available!");
+    if (vpnInstance == null) {
+        LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
+        return;
+    }
     Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
+    RdToElanOpEntry rdToElanOpEntry= getRdToElanOpEntry(broker,vrfTableKey.getRouteDistinguisher(),
+            vrfEntry.getDestPrefix());
+    if (rdToElanOpEntry != null) {
+        if (vpnToDpnList!=null) {
+            for(VpnToDpnList curDpn :  vpnToDpnList) {
+                makeConnectedRoute(curDpn.getDpnId(),vpnInstance.getVpnId(),vrfEntry,vrfTableKey
+                        .getRouteDistinguisher(), null,NwConstants.DEL_FLOW);
+                makeLFibTableEntry(curDpn.getDpnId(),vrfEntry.getLabel(),null,
+                        vrfEntry.getNextHopAddress(),NwConstants.DEL_FLOW);
+                // TODO DeleteTunnelTableEntry();
+            }
+        }
+        //Delete rd-to-elan-op-entry
+        InstanceIdentifier<RdToElanOpEntry> id = getRdToElanOpEntryDataPath(vrfTableKey.getRouteDistinguisher(),
+                vrfEntry.getDestPrefix());
+        MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL,id);
+        FibUtil.releaseId(idManager,FibConstants.VPN_IDPOOL_NAME,
+                FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
+        return;
+    }
+    BigInteger localDpnId = deleteLocalFibEntry(vpnInstance.getVpnId(),
+            vrfTableKey.getRouteDistinguisher(), vrfEntry);
     if (vpnToDpnList != null) {
-      BigInteger localDpnId = deleteLocalFibEntry(vpnInstance.getVpnId(),
-                          vrfTableKey.getRouteDistinguisher(), vrfEntry);
-      for (VpnToDpnList curDpn : vpnToDpnList) {
-        if (!curDpn.getDpnId().equals(localDpnId)) {
-          deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry);
+        for (VpnToDpnList curDpn : vpnToDpnList) {
+            if (!curDpn.getDpnId().equals(localDpnId)) {
+                deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry);
+            }
         }
-      }
-
     }
+    //The flow/group entry has been deleted from config DS; need to clean up associated operational
+    //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
+    cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
   }
 
   public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
@@ -438,17 +643,27 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                                 final VrfEntry vrfEntry) {
     LOG.debug("deleting route "+ vrfEntry.getDestPrefix() + " "+vpnId);
     String rd = vrfTableKey.getRouteDistinguisher();
-    String egressInterface = resolveAdjacency(localDpnId, remoteDpnId, vpnId, vrfEntry);
-    if(egressInterface == null) {
-      LOG.error("Could not get nexthop group id for nexthop: {} in vpn {}",
-                vrfEntry.getNextHopAddress(), rd);
-      LOG.warn("Failed to delete Route: {} in vpn: {}",
-               vrfEntry.getDestPrefix(), rd);
-      return;
+    boolean isRemoteRoute = true;
+    if (localDpnId == null) {
+      // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
+      VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
+      if(localNextHopInfo == null) {
+        //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+        Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+        if (extra_route != null) {
+          localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, extra_route.getNexthopIp());
+        }
+      }
+      if (localNextHopInfo != null) {
+        isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId()));
+      }
+    }
+    if (isRemoteRoute) {
+      makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW);
+      LOG.debug("Successfully delete fib entry for "+ vrfEntry.getDestPrefix() + " vpnId "+vpnId);
+    } else{
+      LOG.debug("Did not delete fib entry rd: {} =, prefix: {} as it is local to dpn {}", rd, vrfEntry.getDestPrefix(), remoteDpnId);
     }
-
-    makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW);
-    LOG.debug("Successfully delete fib entry for "+ vrfEntry.getDestPrefix() + " vpnId "+vpnId);
   }
 
   private long getIpAddress(byte[] rawIpAddress) {
@@ -457,8 +672,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   }
 
   private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
-                                  List<ActionInfo> actionInfos, int addOrRemove) {
-    LOG.trace("makeConnectedRoute: vrfEntry {}",vrfEntry);
+                                  List<InstructionInfo> instructions, int addOrRemove) {    LOG.trace("makeConnectedRoute: vrfEntry {}",vrfEntry);
     String values[] = vrfEntry.getDestPrefix().split("/");
     String ipAddress = values[0];
     int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
@@ -480,21 +694,16 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                               new long[] { 0x0800L }));
 
     if(prefixLength != 0) {
-      matches.add(new MatchInfo(MatchFieldType.ipv4_dst, new long[] {
-          getIpAddress(destPrefix.getAddress()), prefixLength }));
+        matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
+                destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
     }
 
-    List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
-    if(addOrRemove == NwConstants.ADD_FLOW) {
-      instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos));
-    }
-
-    String flowRef = getFlowRef(dpId, L3_FIB_TABLE, rd, destPrefix);
+    String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, destPrefix);
 
     FlowEntity flowEntity;
 
     int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
-    flowEntity = MDSALUtil.buildFlowEntity(dpId, L3_FIB_TABLE, flowRef,
+    flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef,
                                            priority, flowRef, 0, 0,
                                            COOKIE_VM_FIB_TABLE, matches, instructions);
 
@@ -504,34 +713,24 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
       * for or for notification that operational DS write for flows is done. We do not turn on the stats writing for flows,
       * so that notification never comes, so we do not need that wait. Sending the lowest value of wait "1 ms" since 0 wait means
       * wait indefinitely. */
-      // FIXME: sync calls.
-      //mdsalManager.syncInstallFlow(flowEntity, 1);
-      mdsalManager.installFlow(flowEntity);
+      mdsalManager.syncInstallFlow(flowEntity, 1);
     } else {
-      // FIXME: sync calls.
-      // mdsalManager.syncRemoveFlow(flowEntity, 1);
-      mdsalManager.removeFlow(flowEntity);
+      mdsalManager.syncRemoveFlow(flowEntity, 1);
     }
   }
 
-  private void makeLFibTableEntry(BigInteger dpId, long label, long groupId,
+  private void makeLFibTableEntry(BigInteger dpId, long label, List<InstructionInfo> instructions,
                                   String nextHop, int addOrRemove) {
     List<MatchInfo> matches = new ArrayList<MatchInfo>();
     matches.add(new MatchInfo(MatchFieldType.eth_type,
                               new long[] { 0x8847L }));
     matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
 
-    List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
-    List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
-    actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
-    actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
-    instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
-
     // Install the flow entry in L3_LFIB_TABLE
-    String flowRef = getFlowRef(dpId, L3_LFIB_TABLE, label, nextHop);
+    String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, nextHop);
 
     FlowEntity flowEntity;
-    flowEntity = MDSALUtil.buildFlowEntity(dpId, L3_LFIB_TABLE, flowRef,
+    flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef,
                                            DEFAULT_FIB_FLOW_PRIORITY, flowRef, 0, 0,
                                            COOKIE_VM_LFIB_TABLE, matches, instructions);
 
@@ -542,15 +741,11 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
       * so that notification never comes, so we do not need that wait. Sending the lowest value of wait "1 ms" since 0 wait means
       * wait indefinitely. */
 
-      // FIXME:
-      // mdsalManager.syncInstallFlow(flowEntity, 1);
-      mdsalManager.installFlow(flowEntity);
+      mdsalManager.syncInstallFlow(flowEntity, 1);
     } else {
-      // FIXME:
-      // mdsalManager.syncRemoveFlow(flowEntity, 1);
-      mdsalManager.removeFlow(flowEntity);
+      mdsalManager.syncRemoveFlow(flowEntity, 1);
     }
-    LOG.debug("LFIB Entry for dpID {} : label : {} group {} modified successfully {}",dpId, label, groupId );
+      LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",dpId, label, instructions );
   }
 
   private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
@@ -565,12 +760,39 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   public void populateFibOnNewDpn(BigInteger dpnId, long vpnId, String rd) {
     LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
     InstanceIdentifier<VrfTables> id = buildVrfId(rd);
-    Optional<VrfTables> vrfTable = read(LogicalDatastoreType.OPERATIONAL, id);
-    if(vrfTable.isPresent()) {
-      for(VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-        // Passing null as we don't know the dpn
-        // to which prefix is attached at this point
-        createRemoteFibEntry(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry);
+    String lockOnDpnVpn = new String(dpnId.toString()+ vpnId);
+    synchronized (lockOnDpnVpn.intern()) {
+      Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+      if (vrfTable.isPresent()) {
+        for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+            RdToElanOpEntry rdToElanOpEntry = getRdToElanOpEntry(broker, rd,
+                    vrfEntry.getDestPrefix());
+            if (rdToElanOpEntry!= null) {
+                installSubnetRouteInFib(dpnId, rdToElanOpEntry, vpnId, vrfEntry);
+                continue;
+            }
+          // Passing null as we don't know the dpn
+          // to which prefix is attached at this point
+          createRemoteFibEntry(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry);
+        }
+      }
+    }
+  }
+
+  public void populateFibOnDpn(BigInteger dpnId, long vpnId, String rd, String nexthopIp) {
+    LOG.trace("dpn {} for vpn {}, nexthopIp {} : populateFibOnDpn", dpnId, rd, nexthopIp);
+    InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+    String lockOnDpnVpn = new String(dpnId.toString()+ vpnId);
+    synchronized (lockOnDpnVpn.intern()) {
+      Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+      if (vrfTable.isPresent()) {
+        for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+          // Passing null as we don't know the dpn
+          // to which prefix is attached at this point
+          if (nexthopIp == vrfEntry.getNextHopAddress()) {
+            createRemoteFibEntry(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry);
+          }
+        }
       }
     }
   }
@@ -578,12 +800,44 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   public void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd) {
     LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
     InstanceIdentifier<VrfTables> id = buildVrfId(rd);
-    Optional<VrfTables> vrfTable = read(LogicalDatastoreType.OPERATIONAL, id);
-    if(vrfTable.isPresent()) {
-      for(VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-        // Passing null as we don't know the dpn
-        // to which prefix is attached at this point
-        deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry);
+    String lockOnDpnVpn = new String(dpnId.toString()+ vpnId);
+    synchronized (lockOnDpnVpn.intern()) {
+      Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+      if (vrfTable.isPresent()) {
+        for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                        /* Handle subnet routes here */
+            RdToElanOpEntry rdToElanOpEntry= getRdToElanOpEntry(broker, rd,
+                    vrfEntry.getDestPrefix());
+            if (rdToElanOpEntry != null) {
+                LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
+                        dpnId, rd);
+                makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW);
+                makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null,
+                        vrfEntry.getNextHopAddress(),NwConstants.DEL_FLOW);
+                continue;
+            }
+          // Passing null as we don't know the dpn
+          // to which prefix is attached at this point
+          deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry);
+        }
+      }
+    }
+  }
+
+  public void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd, String nexthopIp) {
+    LOG.trace("dpn {} for vpn {}, nexthopIp {} : cleanUpDpnForVpn", dpnId, rd, nexthopIp);
+    InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+    String lockOnDpnVpn = new String(dpnId.toString()+ vpnId);
+    synchronized (lockOnDpnVpn.intern()) {
+      Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+      if (vrfTable.isPresent()) {
+        for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+          // Passing null as we don't know the dpn
+          // to which prefix is attached at this point
+          if (nexthopIp == vrfEntry.getNextHopAddress()) {
+            deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry);
+          }
+        }
       }
     }
   }
@@ -610,14 +864,20 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   }
 
   protected String resolveAdjacency(final BigInteger localDpnId, final BigInteger remoteDpnId,
-                                              final long vpnId, final VrfEntry vrfEntry) {
+                                              final long vpnId, final VrfEntry vrfEntry, String rd) {
     String adjacency = null;
-    LOG.trace("resolveAdjacency called with localdpid{} remotedpid {}, vpnId{}, VrfEntry {}", localDpnId, remoteDpnId, vpnId, vrfEntry);;
+    boolean staticRoute = false;
+    LOG.trace("resolveAdjacency called with localdpid{} remotedpid {}, vpnId{}, VrfEntry {}", localDpnId, remoteDpnId, vpnId, vrfEntry);
     try {
-      adjacency =
+        Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+        if(extra_route != null) {
+            staticRoute = true;
+        }
+
+        adjacency =
           nextHopManager.getRemoteNextHopPointer(localDpnId, remoteDpnId, vpnId,
-                                                 vrfEntry.getDestPrefix(),
-                                                 vrfEntry.getNextHopAddress());
+                  (staticRoute == true) ? extra_route.getNexthopIp() + "/32" : vrfEntry.getDestPrefix(),
+                                                vrfEntry.getNextHopAddress());
     } catch (NullPointerException e) {
       LOG.trace("", e);
     }
@@ -627,7 +887,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
   protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
     InstanceIdentifier<VpnInstanceOpDataEntry> id = InstanceIdentifier.create(VpnInstanceOpData.class).child(
         VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
-    Optional<VpnInstanceOpDataEntry> vpnInstanceOpData = read(LogicalDatastoreType.OPERATIONAL, id);
+    Optional<VpnInstanceOpDataEntry> vpnInstanceOpData = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
     if(vpnInstanceOpData.isPresent()) {
       return vpnInstanceOpData.get();
     }
@@ -637,7 +897,6 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     public void processNodeAdd(BigInteger dpnId) {
         LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId);
         makeTableMissFlow(dpnId, NwConstants.ADD_FLOW);
-        makeProtocolTableFlow(dpnId, NwConstants.ADD_FLOW);
         makeL3IntfTblMissFlow(dpnId, NwConstants.ADD_FLOW);
     }
 
@@ -645,13 +904,13 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
         // Instruction to goto L3 InterfaceTable
         List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
-        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { L3_INTERFACE_TABLE }));
+        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE }));
         List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, L3_LFIB_TABLE,
-                getFlowRef(dpnId, L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW),
+        FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_LFIB_TABLE,
+                getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW),
                 NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
 
-        FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,L3_FIB_TABLE, getFlowRef(dpnId, L3_FIB_TABLE, NwConstants.TABLE_MISS_FLOW),
+        FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE, getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, NwConstants.TABLE_MISS_FLOW),
                 NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow", 0, 0, COOKIE_VM_FIB_TABLE,
                 matches, instructions);
 
@@ -680,13 +939,13 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
     // Instruction to goto L3 InterfaceTable
     List<InstructionInfo> instructions = new ArrayList<>();
-    instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {L3_LFIB_TABLE}));
+    instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
     List<MatchInfo> matches = new ArrayList<MatchInfo>();
     matches.add(new MatchInfo(MatchFieldType.eth_type,
                               new long[] { 0x8847L }));
-    FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, L3_PROTOCOL_TABLE,
-                                                          getFlowRef(dpnId, L3_PROTOCOL_TABLE,
-                                                                     L3_LFIB_TABLE),
+    FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_PROTOCOL_TABLE,
+                                                          getFlowRef(dpnId, NwConstants.L3_PROTOCOL_TABLE,
+                                                                  NwConstants.L3_LFIB_TABLE),
                                                           DEFAULT_FIB_FLOW_PRIORITY,
                                                           "Protocol Table For LFIB",
                                                           0, 0,
@@ -706,7 +965,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     result.add(String.format("   %-7s  %-20s  %-20s  %-7s", "RD", "Prefix", "Nexthop", "Label"));
     result.add("-------------------------------------------------------------------");
     InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
-    Optional<FibEntries> fibEntries = read(LogicalDatastoreType.OPERATIONAL, id);
+    Optional<FibEntries> fibEntries = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
     if (fibEntries.isPresent()) {
       List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
       for (VrfTables vrfTable : vrfTables) {
@@ -730,10 +989,14 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     instructions.add(new InstructionInfo(InstructionType.clear_actions));
     // Instruction to goto L3 InterfaceTable
 
-    instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { LPORT_DISPATCHER_TABLE }));
+    List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
+    actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
+        Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
+    instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+    //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
 
-    FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, L3_INTERFACE_TABLE,
-            getFlowRef(dpnId, L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
+    FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
+            getFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
             NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
     if (addOrRemove == NwConstants.ADD_FLOW) {
       LOG.info("Invoking MDSAL to install L3 interface Table Miss Entries");