Vpnmanager module sync up 41/42241/2
authorKiran N Upadhyaya <kiran.n.upadhyaya@ericsson.com>
Thu, 21 Jul 2016 14:00:39 +0000 (19:30 +0530)
committereupakir <kiran.n.upadhyaya@ericsson.com>
Thu, 21 Jul 2016 19:43:06 +0000 (01:13 +0530)
This commit contains upstreaming of VpnManger module alongwith dependent
commits into fibmanager and neutronvpn manager

Change-Id: I631d9f3652c863180804cdf426d15e6e3ea1cfbc
Signed-off-by: Kiran N Upadhyaya <kiran.n.upadhyaya@ericsson.com>
36 files changed:
vpnservice/bgpmanager/bgpmanager-api/src/main/java/org.opendaylight.netvirt.bgpmanager.api/RouteOrigin.java
vpnservice/fibmanager/fibmanager-api/src/main/java/org/opendaylight/netvirt/fibmanager/api/IFibManager.java
vpnservice/fibmanager/fibmanager-api/src/main/yang/odl-fib.yang
vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManager.java
vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManagerProvider.java
vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibUtil.java
vpnservice/fibmanager/fibmanager-impl/src/test/java/org/opendaylight/netvirt/fibmanager/test/FibManagerTest.java
vpnservice/vpnmanager/vpnmanager-api/pom.xml
vpnservice/vpnmanager/vpnmanager-api/src/main/java/org/opendaylight/netvirt/vpnmanager/api/IVpnManager.java
vpnservice/vpnmanager/vpnmanager-api/src/main/yang/inter-vpn-link.yang
vpnservice/vpnmanager/vpnmanager-api/src/main/yang/l3vpn.yang
vpnservice/vpnmanager/vpnmanager-api/src/main/yang/odl-l3vpn.yang
vpnservice/vpnmanager/vpnmanager-api/src/main/yang/vpn-rpc.yang
vpnservice/vpnmanager/vpnmanager-impl/pom.xml
vpnservice/vpnmanager/vpnmanager-impl/src/main/config/default-config.xml
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpConstants.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpNotificationHandler.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpScheduler.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/InterfaceStateChangeListener.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/MacEntry.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/SubnetOpDpnManager.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/SubnetRoutePacketInHandler.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnConstants.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManager.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnRpcServiceImpl.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnUtil.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnserviceProvider.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkListener.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkNodeAddTask.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkNodeListener.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkUtil.java [new file with mode: 0644]
vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/opendaylight/params/xml/ns/yang/vpnservice/impl/rev150216/VpnserviceImplModule.java
vpnservice/vpnmanager/vpnmanager-impl/src/main/yang/vpnservice-impl.yang
vpnservice/vpnmanager/vpnmanager-impl/src/test/java/org/opendaylight/netvirt/vpnmanager/test/VpnSubnetRouteHandlerTest.java

index e0d068ff04d1da36060be8b79866374314cb8bf7..c410676e12050e8c515aa49f40ba011a4b66f1d8 100644 (file)
@@ -18,6 +18,7 @@ public enum RouteOrigin {
     CONNECTED("directly connected", "c"),
     STATIC(   "static",             "s"),
     INTERVPN( "inter-vpn link",     "l"),
+    SELF_IMPORTED( "self imported route","selfimp"),
     BGP(      "bgp",                "b"),
     IGP(      "igp",                "i");
 
@@ -43,6 +44,9 @@ public enum RouteOrigin {
             case "l":
                 origin = INTERVPN;
                 break;
+            case "selfimp":
+                origin = SELF_IMPORTED;
+                break;
             case "b":
                 origin = BGP;
                 break;
index 4bca84fbca9830dde73618f4beccfc63d95149f9..51c3c38198f7c137002c5659610d36a9302670b9 100644 (file)
@@ -13,8 +13,10 @@ import java.util.List;
 
 public interface IFibManager {
     void populateFibOnNewDpn(BigInteger dpnId, long vpnId, String rd);
-    void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd, String nextHopIp);
-    void populateFibOnDpn(BigInteger localDpnId, BigInteger destDpnId, long vpnId, String rd, String nextHopIp);
+    void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd,
+                          String localNextHopIp, String remoteNextHopIp);
+    void populateFibOnDpn(BigInteger localDpnId, long vpnId, String rd,
+                          String localNextHopIp, String remoteNextHopIp);
     void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd);
     List<String> printFibEntries();
 
@@ -27,6 +29,9 @@ public interface IFibManager {
     void writeConfTransTypeConfigDS();
     String getReqTransType();
     String getTransportTypeStr(String tunType);
-    void handleRemoteRoute(boolean action, BigInteger localDpnId, BigInteger remoteDpnId,
-                           long vpnId, String rd, String destPrefix, String nextHopIp);
+    void handleRemoteRoute(boolean action, BigInteger localDpnId,
+                           BigInteger remoteDpnId, long vpnId,
+                           String rd, String destPrefix,
+                           String localNextHopIp,
+                           String remoteNextHopIP);
 }
index 10d70f84cccdaed241d29a213b12c93b3ce0d652..360a5c10fdc43a37bcebdfe89fe7b48701bcc539 100644 (file)
@@ -52,7 +52,24 @@ module odl-fib {
         }
 
         container ipv4Table{
-            uses  ipv4Entries;
+            uses ipv4Entries;
         }
     }
+
+    container label-route-map {
+         config false;
+         list label-route-info {
+            key label;
+            leaf label { type uint32; }
+            leaf dpn-id { type uint64; }
+            leaf prefix { type string; }
+            leaf-list next-hop-ip-list { type string; }
+            leaf-list vpn-instance-list { type string; }
+            leaf parent-vpnid { type uint32; }
+            leaf vpn-interface-name { type string; }
+            leaf elan-tag { type uint32; }
+            leaf is-subnet-route { type boolean; }
+            leaf parent-vpn-rd { type string; }
+         }
+    }
 }
\ No newline at end of file
index 61db4a4be96c61f634d2726c36e6b2d588ba04ce..3276d85dcf1c87679c092a881432739dc7ef370a 100644 (file)
@@ -58,21 +58,30 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnToExtraroute;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 
 
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn
         .link.states.InterVpnLinkState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn
@@ -105,199 +114,199 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class FibManager extends AbstractDataChangeListener<VrfEntry> implements AutoCloseable{
-    private static final Logger LOG = LoggerFactory.getLogger(FibManager.class);
-    private static final String FLOWID_PREFIX = "L3.";
-    private ListenerRegistration<DataChangeListener> listenerRegistration;
-    private final DataBroker broker;
-    private IMdsalApiManager mdsalManager;
-    private IVpnManager vpnmanager;
-    private NexthopManager nextHopManager;
-    private ItmRpcService itmManager;
-
-    private OdlInterfaceRpcService interfaceManager;
-    private IdManagerService idManager;
-    private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000002", 16);
-    private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
-    private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
-    private static final int LFIB_INTERVPN_PRIORITY = 1;
-    private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
-    private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
-    public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
-
-
-    public FibManager(final DataBroker db) {
-        super(VrfEntry.class);
-        broker = db;
-        registerListener(db);
-    }
-
-    @Override
-    public void close() throws Exception {
-        if (listenerRegistration != null) {
-            try {
-                listenerRegistration.close();
-            } catch (final Exception e) {
-                LOG.error("Error when cleaning up DataChangeListener.", e);
-            }
-            listenerRegistration = null;
-        }
-        LOG.info("Fib Manager Closed");
-    }
-
-    public void setNextHopManager(NexthopManager nextHopManager) {
-        this.nextHopManager = nextHopManager;
-    }
+  private static final Logger LOG = LoggerFactory.getLogger(FibManager.class);
+  private static final String FLOWID_PREFIX = "L3.";
+  private ListenerRegistration<DataChangeListener> listenerRegistration;
+  private final DataBroker broker;
+  private IMdsalApiManager mdsalManager;
+  private IVpnManager vpnmanager;
+  private NexthopManager nextHopManager;
+  private ItmRpcService itmManager;
+
+  private OdlInterfaceRpcService interfaceManager;
+  private IdManagerService idManager;
+  private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000002", 16);
+  private static final BigInteger COOKIE_VM_FIB_TABLE =  new BigInteger("8000003", 16);
+  private static final int DEFAULT_FIB_FLOW_PRIORITY = 10;
+  private static final int LFIB_INTERVPN_PRIORITY = 1;
+  private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16);
+  private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0);
+  public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
+
+
+  public FibManager(final DataBroker db) {
+    super(VrfEntry.class);
+    broker = db;
+    registerListener(db);
+  }
+
+  @Override
+  public void close() throws Exception {
+    if (listenerRegistration != null) {
+      try {
+        listenerRegistration.close();
+      } catch (final Exception e) {
+        LOG.error("Error when cleaning up DataChangeListener.", e);
+      }
+      listenerRegistration = null;
+    }
+    LOG.info("Fib Manager Closed");
+  }
+
+  public void setNextHopManager(NexthopManager nextHopManager) {
+    this.nextHopManager = nextHopManager;
+  }
     public NexthopManager getNextHopManager() {
         return this.nextHopManager;
     }
 
-    public void setMdsalManager(IMdsalApiManager mdsalManager) {
-        this.mdsalManager = mdsalManager;
-    }
+  public void setMdsalManager(IMdsalApiManager mdsalManager) {
+    this.mdsalManager = mdsalManager;
+  }
 
-    public void setVpnmanager(IVpnManager vpnmanager) {
-        this.vpnmanager = vpnmanager;
-    }
+  public void setVpnmanager(IVpnManager vpnmanager) {
+    this.vpnmanager = vpnmanager;
+  }
 
-    public void setITMRpcService(ItmRpcService itmManager) {
-        this.itmManager = itmManager;
-    }
+  public void setITMRpcService(ItmRpcService itmManager) {
+      this.itmManager = itmManager;
+  }
 
-    public void setInterfaceManager(OdlInterfaceRpcService ifManager) {
-        this.interfaceManager = ifManager;
-    }
+  public void setInterfaceManager(OdlInterfaceRpcService ifManager) {
+      this.interfaceManager = ifManager;
+  }
 
-    public void setIdManager(IdManagerService idManager) {
+  public void setIdManager(IdManagerService idManager) {
         this.idManager = idManager;
-    }
-
-    private void registerListener(final DataBroker db) {
-        try {
-            listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
-                    getWildCardPath(), FibManager.this, DataChangeScope.SUBTREE);
-        } catch (final Exception e) {
-            LOG.error("FibManager DataChange listener registration fail!", e);
-            throw new IllegalStateException("FibManager registration Listener failed.", e);
-        }
-    }
-
-
-    private InstanceIdentifier<VrfEntry> getWildCardPath() {
-        return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
-    }
-
-
-    @Override
-    protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
-        Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
-        String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
-        LOG.info("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
-        createFibEntries(identifier, vrfEntry);
-        LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
-        leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
-    }
-
-    @Override
-    protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
-        Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
-        String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
-        LOG.info("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
-        deleteFibEntries(identifier, vrfEntry);
-        LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
-                rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
-        leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
-    }
-
-    @Override
-    protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
-        Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
-        if (original.getAugmentation(SubnetRoute.class) != null && update.getAugmentation(SubnetRoute.class) == null)
-            return;
-        String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
-        LOG.info("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
-                rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
-        createFibEntries(identifier, update);
-        LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
-                rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
-    }
-
-    private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
-        final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
-
-        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
-        Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
-        Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
-
-        final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
-        final Long vpnId = vpnInstance.getVpnId();
-        final String rd = vrfTableKey.getRouteDistinguisher();
-        SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
-        if (subnetRoute != null) {
-            final long elanTag = subnetRoute.getElantag();
-            LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
-                    rd, vrfEntry.getDestPrefix(), elanTag);
-            if (vpnToDpnList != null) {
-                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
-                        new Callable<List<ListenableFuture<Void>>>() {
-                            @Override
-                            public List<ListenableFuture<Void>> call() throws Exception {
-                                WriteTransaction tx = broker.newWriteOnlyTransaction();
-                                for (final VpnToDpnList curDpn : vpnToDpnList) {
-                                    installSubnetRouteInFib(curDpn.getDpnId(),elanTag, rd, vpnId.longValue(), vrfEntry, tx);
-                                }
-                                List<ListenableFuture<Void>> futures = new ArrayList<>();
-                                futures.add(tx.submit());
-                                return futures;
-                            }
-                        });
-            }
-            return;
-        }
-
-        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
-            // When it is a leaked route, the LFIB and FIB goes a bit different.
-            installInterVpnRouteInLFib(rd, vrfEntry);
-            return;
-        }
-
-        final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
-
-        if (vpnToDpnList != null) {
-            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            WriteTransaction tx = broker.newWriteOnlyTransaction();
-                            for (VpnToDpnList vpnDpn : vpnToDpnList) {
-                                if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
-                                    createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                }
-                            }
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            futures.add(tx.submit());
-                            return futures;
-                        }
-                    });
-        }
-
-        Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(broker, rd);
-        if ( vpnUuid.isPresent() ) {
-            Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(broker, vpnUuid.get());
-            if ( interVpnLink.isPresent() ) {
-                String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
-                if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid.get(), interVpnLink.get()) ) {
-                    // This is an static route that points to the other endpoint of an InterVpnLink
-                    // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
-                    installRouteInInterVpnLink(interVpnLink.get(), rd, vrfEntry, vpnId);
-                }
-            }
-        }
-    }
+  }
+
+  private void registerListener(final DataBroker db) {
+    try {
+      listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
+                                                           getWildCardPath(), FibManager.this, DataChangeScope.SUBTREE);
+    } catch (final Exception e) {
+      LOG.error("FibManager DataChange listener registration fail!", e);
+      throw new IllegalStateException("FibManager registration Listener failed.", e);
+    }
+  }
+
+
+  private InstanceIdentifier<VrfEntry> getWildCardPath() {
+    return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class);
+  }
+
+
+  @Override
+  protected void add(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
+      Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
+      String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
+      LOG.info("ADD: Adding Fib Entry rd {} prefix {} nexthop {} label {}",
+               rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
+      createFibEntries(identifier, vrfEntry);
+      LOG.info("ADD: Added Fib Entry rd {} prefix {} nexthop {} label {}",
+               rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
+      leakRouteIfNeeded(identifier, vrfEntry, NwConstants.ADD_FLOW);
+  }
+
+  @Override
+  protected void remove(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry) {
+    Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty.");
+    String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
+    LOG.info("REMOVE: Removing Fib Entry rd {} prefix {} nexthop {} label {}",
+             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
+    deleteFibEntries(identifier, vrfEntry);
+    LOG.info("REMOVE: Removed Fib Entry rd {} prefix {} nexthop {} label {}",
+             rd, vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), vrfEntry.getLabel());
+    leakRouteIfNeeded(identifier, vrfEntry, NwConstants.DEL_FLOW);
+  }
+
+  @Override
+  protected void update(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update) {
+    Preconditions.checkNotNull(update, "VrfEntry should not be null or empty.");
+    if (original.getAugmentation(SubnetRoute.class) != null && update.getAugmentation(SubnetRoute.class) == null)
+        return;
+    String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher();
+    LOG.info("UPDATE: Updating Fib Entries to rd {} prefix {} nexthop {} label {}",
+             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
+    createFibEntries(identifier, update);
+    LOG.info("UPDATE: Updated Fib Entries to rd {} prefix {} nexthop {} label {}",
+             rd, update.getDestPrefix(), update.getNextHopAddressList(), update.getLabel());
+  }
+
+  private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
+      final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
+
+      final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
+      Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
+      Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
+
+      final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
+      final Long vpnId = vpnInstance.getVpnId();
+      final String rd = vrfTableKey.getRouteDistinguisher();
+      SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+      if (subnetRoute != null) {
+          final long elanTag = subnetRoute.getElantag();
+          LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
+                  rd, vrfEntry.getDestPrefix(), elanTag);
+          if (vpnToDpnList != null) {
+              DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+              dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
+                      new Callable<List<ListenableFuture<Void>>>() {
+                          @Override
+                          public List<ListenableFuture<Void>> call() throws Exception {
+                              WriteTransaction tx = broker.newWriteOnlyTransaction();
+                              for (final VpnToDpnList curDpn : vpnToDpnList) {
+                                  installSubnetRouteInFib(curDpn.getDpnId(),elanTag, rd, vpnId.longValue(), vrfEntry, tx);
+                              }
+                              List<ListenableFuture<Void>> futures = new ArrayList<>();
+                              futures.add(tx.submit());
+                              return futures;
+                          }
+                      });
+          }
+          return;
+      }
+
+      if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) {
+          // When it is a leaked route, the LFIB and FIB goes a bit different.
+          installInterVpnRouteInLFib(rd, vrfEntry);
+          return;
+      }
+
+      final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
+
+      if (vpnToDpnList != null) {
+          DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+          dataStoreCoordinator.enqueueJob("FIB"+rd.toString()+vrfEntry.getDestPrefix(),
+                  new Callable<List<ListenableFuture<Void>>>() {
+                      @Override
+                      public List<ListenableFuture<Void>> call() throws Exception {
+                          WriteTransaction tx = broker.newWriteOnlyTransaction();
+                          for (VpnToDpnList vpnDpn : vpnToDpnList) {
+                              if ( !localDpnIdList.contains(vpnDpn.getDpnId())) {
+                                  createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
+                              }
+                          }
+                          List<ListenableFuture<Void>> futures = new ArrayList<>();
+                          futures.add(tx.submit());
+                          return futures;
+                      }
+                  });
+      }
+
+      Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(broker, rd);
+      if ( vpnUuid.isPresent() ) {
+          Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(broker, vpnUuid.get());
+          if ( interVpnLink.isPresent() ) {
+              String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
+              if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid.get(), interVpnLink.get()) ) {
+                  // This is an static route that points to the other endpoint of an InterVpnLink
+                  // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
+                  installRouteInInterVpnLink(interVpnLink.get(), rd, vrfEntry, vpnId);
+              }
+          }
+      }
+  }
 
     /*
      * Returns true if the specified nexthop is the other endpoint in an
@@ -305,11 +314,11 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
      */
     private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) {
         return
-                interVpnLink != null
-                        && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
-                        && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
-                        || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
-                        && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
+            interVpnLink != null
+            && (   (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid)
+                     && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop))
+                || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid )
+                     && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) );
     }
 
 
@@ -322,14 +331,18 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         String rd = vrfTableKey.getRouteDistinguisher();
         VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd);
-        Preconditions.checkNotNull(vpnInstance,
-                "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
-        Preconditions.checkNotNull(vpnInstance.getVpnId(),
-                "Vpn Instance with vrf " + vpnInstance.getVrfId() + " has null vpnId!");
-
+        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) {
+            if (vpnInstance == null) {
+                LOG.error("Vpn Instance not available for external route with prefix {} label {} nexthop {}. Returning...", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList());
+                return;
+            }
+        } else {
+            Preconditions.checkNotNull(vpnInstance,
+                    "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher());
+        }
         String vpnUuid = vpnInstance.getVpnInstanceName();
         Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(),
-                "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
+                                    "Could not find suitable VPN UUID for Route-Distinguisher=" + rd);
 
         // if the new vrfEntry has been learned by Quagga BGP, its necessary to check if it's
         // there an interVpnLink for the involved vpn in order to make learn the new route to
@@ -337,8 +350,8 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink.
         Optional<InterVpnLink> interVpnLink =
-                (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(broker, rd)
-                        : FibUtil.getInterVpnLinkByRd(broker, rd);
+            (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(broker, rd)
+                                                  : FibUtil.getInterVpnLinkByRd(broker, rd);
         if ( !interVpnLink.isPresent() ) {
             LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd);
             return;
@@ -347,31 +360,31 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         // Ok, at this point everything is ready for the leaking/removal... but should it be performed?
         // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled.
         boolean proceed = (addOrRemove == NwConstants.DEL_FLOW )
-                || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
-                && interVpnLink.get().isBgpRoutesLeaking() );
+                          || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP
+                               && interVpnLink.get().isBgpRoutesLeaking() );
 
         if ( proceed ) {
             String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) )
-                    ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
-                    : vpnUuid;
+                                     ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue()
+                                     : vpnUuid;
 
             String dstVpnRd = FibUtil.getVpnRd(broker, theOtherVpnId);
             String endpointIp = vrfEntry.getNextHopAddressList().get(0);
 
             InstanceIdentifier<VrfEntry> vrfEntryIidInOtherVpn =
-                    InstanceIdentifier.builder(FibEntries.class)
-                            .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
-                            .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
-                            .build();
+                InstanceIdentifier.builder(FibEntries.class)
+                                  .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
+                                  .child(VrfEntry.class, new VrfEntryKey(vrfEntry.getDestPrefix()))
+                                  .build();
             if ( addOrRemove == NwConstants.ADD_FLOW ) {
                 LOG.info("Leaking route (destination={}, nexthop={}) from Vrf={} to Vrf={}",
-                        vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
+                         vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), rd, dstVpnRd);
                 String key = rd + FibConstants.SEPARATOR + vrfEntry.getDestPrefix();
                 long label = FibUtil.getUniqueId(idManager, FibConstants.VPN_IDPOOL_NAME, key);
                 VrfEntry newVrfEntry = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(Arrays.asList(endpointIp))
-                        .setLabel(label)
-                        .setOrigin(RouteOrigin.INTERVPN.getValue())
-                        .build();
+                                                                    .setLabel(label)
+                                                                    .setOrigin(RouteOrigin.INTERVPN.getValue())
+                                                                    .build();
                 MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryIidInOtherVpn, newVrfEntry);
             } else {
                 LOG.info("Removing leaked vrfEntry={}", vrfEntryIidInOtherVpn.toString());
@@ -380,6 +393,28 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         }
     }
 
+    private Prefixes updateVpnReferencesInLri(LabelRouteInfo lri, String vpnInstanceName, boolean isPresentInList) {
+        LOG.info("updating LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
+        PrefixesBuilder prefixBuilder = new PrefixesBuilder();
+        prefixBuilder.setDpnId(lri.getDpnId());
+        prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
+        prefixBuilder.setIpAddress(lri.getPrefix());
+        // Increment the refCount here
+        InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
+                .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)lri.getLabel())).build();
+        LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri);
+        if (!isPresentInList) {
+            LOG.debug("vpnName {} is not present in LRI with label {}..", vpnInstanceName, lri.getLabel());
+            List<String> vpnInstanceNames = lri.getVpnInstanceList();
+            vpnInstanceNames.add(vpnInstanceName);
+            builder.setVpnInstanceList(vpnInstanceNames);
+            FibUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
+        } else {
+            LOG.debug("vpnName {} is present in LRI with label {}..", vpnInstanceName, lri.getLabel());
+        }
+        return prefixBuilder.build();
+    }
+
     private void installSubnetRouteInFib(final BigInteger dpnId, final long elanTag, final String rd,
                                          final long vpnId, final VrfEntry vrfEntry, WriteTransaction tx){
         Boolean wrTxPresent = true;
@@ -387,23 +422,43 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
             wrTxPresent = false;
             tx = broker.newWriteOnlyTransaction();
         }
+        synchronized (vrfEntry.getLabel().toString().intern()) {
+            LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+            if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
+                    vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+
+                if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
+                    Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
+                    if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                        String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                        if (!lri.getVpnInstanceList().contains(vpnInstanceName)) {
+                            updateVpnReferencesInLri(lri, vpnInstanceName, false);
+                        }
+                    }
+                }
+                LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
+                        vrfEntry.getLabel(), lri.getVpnInterfaceName(), lri.getDpnId());
+            }
+        }
         final List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
         BigInteger subnetRouteMeta =  ((BigInteger.valueOf(elanTag)).shiftLeft(32)).or((BigInteger.valueOf(vpnId)));
         instructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
         instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
         makeConnectedRoute(dpnId,vpnId,vrfEntry,rd,instructions,NwConstants.ADD_FLOW, tx);
 
-        List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
-        // reinitialize instructions list for LFIB Table
-        final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
+        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+            List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
+            // reinitialize instructions list for LFIB Table
+            final List<InstructionInfo> LFIBinstructions = new ArrayList<InstructionInfo>();
 
-        actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
-        LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
-        LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
-        LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
+            actionsInfos.add(new ActionInfo(ActionType.pop_mpls, new String[]{}));
+            LFIBinstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+            LFIBinstructions.add(new InstructionInfo(InstructionType.write_metadata,  new BigInteger[] { subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE }));
+            LFIBinstructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_SUBNET_ROUTE_TABLE }));
 
-        makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
-        if(!wrTxPresent ){
+            makeLFibTableEntry(dpnId,vrfEntry.getLabel(), LFIBinstructions, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
+        }
+        if (!wrTxPresent ) {
             tx.submit();
         }
     }
@@ -424,41 +479,41 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         for ( InterVpnLink interVpnLink : interVpnLinks ) {
             boolean vpnIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName);
             boolean vpnIs2ndEndpoint = !vpnIs1stEndpoint
-                    && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
+                                        && interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(vpnName);
             if ( vpnIs1stEndpoint || vpnIs2ndEndpoint ) {
                 interVpnLinkFound = true;
 
                 Optional<InterVpnLinkState> vpnLinkState = FibUtil.getInterVpnLinkState(broker, interVpnLink.getName());
                 if ( !vpnLinkState.isPresent()
-                        || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
+                     || !vpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
                     LOG.warn("InterVpnLink {}, linking VPN {} and {}, is not in Active state",
-                            interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
-                            interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
+                             interVpnLink.getName(), interVpnLink.getFirstEndpoint().getVpnUuid().getValue(),
+                             interVpnLink.getSecondEndpoint().getVpnUuid().getValue() );
                     return;
                 }
 
                 List<BigInteger> targetDpns =
-                        ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
-                                : vpnLinkState.get().getSecondEndpointState().getDpId();
+                    ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId()
+                                         : vpnLinkState.get().getSecondEndpointState().getDpId();
                 int lportTag =
-                        ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
-                                : vpnLinkState.get().getFirstEndpointState().getLportTag();
+                    ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag()
+                                         : vpnLinkState.get().getFirstEndpointState().getLportTag();
 
                 for ( BigInteger dpId : targetDpns ) {
                     List<ActionInfo> actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}));
 
                     BigInteger[] metadata = new BigInteger[] {
-                            MetaDataUtil.getMetaDataForLPortDispatcher(lportTag, FibConstants.L3VPN_SERVICE_IDENTIFIER),
-                            MetaDataUtil.getMetaDataMaskForLPortDispatcher()
+                        MetaDataUtil.getMetaDataForLPortDispatcher(lportTag, FibConstants.L3VPN_SERVICE_IDENTIFIER),
+                        MetaDataUtil.getMetaDataMaskForLPortDispatcher()
                     };
                     List<InstructionInfo> instructions =
-                            Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
-                                    new InstructionInfo(InstructionType.write_metadata, metadata),
-                                    new InstructionInfo(InstructionType.goto_table,
-                                            new long[] { NwConstants.L3_INTERFACE_TABLE }));
+                        Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos),
+                                      new InstructionInfo(InstructionType.write_metadata, metadata),
+                                      new InstructionInfo(InstructionType.goto_table,
+                                                          new long[] { NwConstants.L3_INTERFACE_TABLE }));
 
                     makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY,
-                            NwConstants.ADD_FLOW, null);
+                                       NwConstants.ADD_FLOW, null);
                 }
 
                 break;
@@ -467,7 +522,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         if ( !interVpnLinkFound ) {
             LOG.warn("VrfEntry=[prefix={} label={} nexthop={}] for VPN {} has origin INTERVPN but no InterVpnLink could be found",
-                    vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
+                     vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), rd);
         }
     }
 
@@ -477,7 +532,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                                             final VrfEntry vrfEntry, long vpnTag) {
         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
-                && vrfEntry.getNextHopAddressList().size() == 1);
+                                    && vrfEntry.getNextHopAddressList().size() == 1);
 
         // After having received a static route, we should check if the vpn is part of an inter-vpn-link.
         // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table
@@ -489,7 +544,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         }
         if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) {
             LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active",
-                    vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList().get(0), interVpnLink.getName());
+                     vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList().get(0), interVpnLink.getName());
             return;
         }
 
@@ -497,22 +552,22 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         // Everything Ok
         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
         List<BigInteger> targetDpns =
-                vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
-                        : interVpnLinkState.get().getSecondEndpointState().getDpId();
+            vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
+                               : interVpnLinkState.get().getSecondEndpointState().getDpId();
 
         Integer otherEndpointlportTag =
-                vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
-                        : interVpnLinkState.get().getFirstEndpointState().getLportTag();
+            vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag()
+                               : interVpnLinkState.get().getFirstEndpointState().getLportTag();
 
         BigInteger[] metadata = new BigInteger[] {
-                MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag,
-                        FibConstants.L3VPN_SERVICE_IDENTIFIER),
-                MetaDataUtil.getMetaDataMaskForLPortDispatcher()
-        };
+                        MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag,
+                                                                   FibConstants.L3VPN_SERVICE_IDENTIFIER),
+                        MetaDataUtil.getMetaDataMaskForLPortDispatcher()
+                    };
         List<Instruction> instructions =
-                Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
-                        new InstructionInfo(InstructionType.goto_table,
-                                new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
+            Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0),
+                          new InstructionInfo(InstructionType.goto_table,
+                                              new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1));
 
         String values[] = vrfEntry.getDestPrefix().split("/");
         String destPrefixIpAddress = values[0];
@@ -520,20 +575,20 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         List<MatchInfo> matches = new ArrayList<>();
         matches.add(new MatchInfo(MatchFieldType.metadata,
-                new BigInteger[] { BigInteger.valueOf(vpnTag),
-                        MetaDataUtil.METADATA_MASK_VRFID }));
+                                  new BigInteger[] { BigInteger.valueOf(vpnTag),
+                                                     MetaDataUtil.METADATA_MASK_VRFID }));
         matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 }));
 
         if (prefixLength != 0) {
             matches.add(new MatchInfo(MatchFieldType.ipv4_destination,
-                    new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
+                                      new String[] { destPrefixIpAddress, Integer.toString(prefixLength) }));
         }
 
         int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
         String nextHop = vrfEntry.getNextHopAddressList().get(0);
         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
-                COOKIE_VM_FIB_TABLE, matches, instructions);
+                                                 COOKIE_VM_FIB_TABLE, matches, instructions);
 
         for ( BigInteger dpId : targetDpns ) {
             mdsalManager.installFlow(dpId, flowEntity);
@@ -545,7 +600,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null");
         Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null
-                && vrfEntry.getNextHopAddressList().size() == 1);
+                                    && vrfEntry.getNextHopAddressList().size() == 1);
 
         Optional<InterVpnLinkState> interVpnLinkState = FibUtil.getInterVpnLinkState(broker, interVpnLink.getName());
         if ( !interVpnLinkState.isPresent() ) {
@@ -556,14 +611,14 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         // Everything Ok
         boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid);
         List<BigInteger> targetDpns =
-                vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
-                        : interVpnLinkState.get().getSecondEndpointState().getDpId();
+            vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId()
+                               : interVpnLinkState.get().getSecondEndpointState().getDpId();
 
         String nextHop = vrfEntry.getNextHopAddressList().get(0);
         String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop);
         FlowKey flowKey = new FlowKey(new FlowId(flowRef));
         Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE)
-                .setFlowName(flowRef).build();
+                                     .setFlowName(flowRef).build();
 
         for ( BigInteger dpId : targetDpns ) {
             mdsalManager.removeFlow(dpId, flow);
@@ -614,136 +669,208 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
             dpns.add(dpn.getDpnId());
         }
 
-        return dpns;
-    }
+      return dpns;
+  }
 
-    private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
-        List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
-        Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
-        String localNextHopIP = vrfEntry.getDestPrefix();
+  private List<BigInteger> createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
+    List<BigInteger> returnLocalDpnId = new ArrayList<BigInteger>();
+    Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
+    String localNextHopIP = vrfEntry.getDestPrefix();
 
+    if (localNextHopInfo == null) {
+        //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+        Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+        if (extraRoute != null) {
+            for (String nextHopIp : extraRoute.getNexthopIpList()) {
+                LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
+                if (nextHopIp != null) {
+                    localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
+                    localNextHopIP = nextHopIp + "/32";
+                    BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
+                    returnLocalDpnId.add(dpnId);
+                }
+            }
+        }
         if (localNextHopInfo == null) {
-            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
-            Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            if (extraRoute != null) {
-                //FIXME:Suraj
-//                for (String nextHopIp : extraRoute.getNexthopIpList()) {
-//                    LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
-//                    if (nextHopIp != null) {
-//                        localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
-//                        localNextHopIP = nextHopIp + "/32";
-//                        BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd,
-//                        vrfEntry);
-//                        returnLocalDpnId.add(dpnId);
-//                    }
-//                }
+            /* imported routes case */
+            synchronized (vrfEntry.getLabel().toString().intern()) {
+                LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+                if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
+                        vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+                    if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
+                        Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
+                        if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                            String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                            if (lri.getVpnInstanceList().contains(vpnInstanceName)) {
+                                localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true);
+                            } else {
+                                localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, false);
+                            }
+                        }
+                    }
+                    localNextHopIP = lri.getPrefix();
+                    LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
+                            vrfEntry.getLabel(), localNextHopInfo.getVpnInterfaceName(), lri.getDpnId());
+                    BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, lri.getParentVpnid());
+                    returnLocalDpnId.add(dpnId);
+                }
             }
-        } else {
-            BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry);
-            returnLocalDpnId.add(dpnId);
         }
-
-        return returnLocalDpnId;
+    } else {
+        BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId);
+        returnLocalDpnId.add(dpnId);
+    }
+
+     return returnLocalDpnId;
+  }
+
+   private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
+                                               final VrfEntry vrfEntry, Long parentVpnId){
+       if (localNextHopInfo != null) {
+           final BigInteger dpnId = localNextHopInfo.getDpnId();
+           if (!isVpnPresentInDpn(rd, dpnId)) {
+               return BigInteger.ZERO;
+           }
+
+           final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
+
+           List<ActionInfo> actionsInfos =
+               Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
+           final List<InstructionInfo> instructions =
+               Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
+           actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
+                   new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
+           final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
+           if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+               LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
+                       dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
+           } else {
+               LOG.debug("Route with rd {} prefix {} label {} nexthop {} for vpn {} is an imported route. LFib and Terminating table entries will not be created.", rd, vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpnId);
+           }
+           DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+           dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
+                   new Callable<List<ListenableFuture<Void>>>() {
+                       @Override
+                       public List<ListenableFuture<Void>> call() throws Exception {
+                           WriteTransaction tx = broker.newWriteOnlyTransaction();
+                           makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
+                           if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                               makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
+                               makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
+                           }
+                           List<ListenableFuture<Void>> futures = new ArrayList<>();
+                           futures.add(tx.submit());
+                           return futures;
+                       }
+                   });
+           return dpnId;
+       }
+       return BigInteger.ZERO;
+   }
+
+   private boolean isVpnPresentInDpn(String rd, BigInteger dpnId)  {
+       InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
+       Optional<VpnToDpnList> dpnInVpn = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
+       if (dpnInVpn.isPresent()) {
+           return true;
+       }
+       return false;
+   }
+
+    private LabelRouteInfo getLabelRouteInfo(Long label) {
+        InstanceIdentifier<LabelRouteInfo>lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
+                .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
+        Optional<LabelRouteInfo> opResult = read(broker, LogicalDatastoreType.OPERATIONAL, lriIid);
+        if (opResult.isPresent()) {
+            return opResult.get();
+        }
+        return null;
     }
 
-    private BigInteger checkCreateLocalFibEntry(Prefixes localNextHopInfo, String localNextHopIP, final Long vpnId, final String rd,
-                                                final VrfEntry vrfEntry){
-        if (localNextHopInfo != null) {
-            final BigInteger dpnId = localNextHopInfo.getDpnId();
-            final long groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP);
-
-            List<ActionInfo> actionsInfos =
-                    Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)}));
-            final List<InstructionInfo> instructions =
-                    Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
-
-            actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}),
-                    new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) );
-            final List<InstructionInfo> lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos));
-
-            LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}",
-                    dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel());
-
-            DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-            dataStoreCoordinator.enqueueJob("FIB"+vpnId.toString()+dpnId.toString()+vrfEntry.getDestPrefix(),
-                    new Callable<List<ListenableFuture<Void>>>() {
-                        @Override
-                        public List<ListenableFuture<Void>> call() throws Exception {
-                            WriteTransaction tx = broker.newWriteOnlyTransaction();
-
-                            makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx);
-                            makeLFibTableEntry(dpnId, vrfEntry.getLabel(), lfibinstructions , DEFAULT_FIB_FLOW_PRIORITY, NwConstants.ADD_FLOW, tx);
-                            makeTunnelTableEntry(dpnId, vrfEntry.getLabel(), groupId, tx);
-                            List<ListenableFuture<Void>> futures = new ArrayList<>();
-                            futures.add(tx.submit());
-                            return futures;
-                        }
-                    });
-            return dpnId;
+    private boolean deleteLabelRouteInfo(LabelRouteInfo lri, String vpnInstanceName) {
+        LOG.info("deleting LRI : for label {} vpninstancename {}", lri.getLabel(), vpnInstanceName);
+        InstanceIdentifier<LabelRouteInfo> lriId = InstanceIdentifier.builder(LabelRouteMap.class)
+                .child(LabelRouteInfo.class, new LabelRouteInfoKey((long) lri.getLabel())).build();
+        if (lri == null) {
+            return true;
         }
-        return BigInteger.ZERO;
+        List<String> vpnInstancesList = lri.getVpnInstanceList();
+        if (vpnInstancesList.contains(vpnInstanceName)) {
+            LOG.debug("vpninstance {} name is present", vpnInstanceName);
+            vpnInstancesList.remove(vpnInstanceName);
+        }
+        if (vpnInstancesList.size() == 0) {
+            LOG.debug("deleting LRI instance object for label {}", lri.getLabel());
+            FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, lriId);
+            return true;
+        } else {
+            LOG.debug("updating LRI instance object for label {}", lri.getLabel());
+            LabelRouteInfoBuilder builder = new LabelRouteInfoBuilder(lri).setVpnInstanceList(vpnInstancesList);
+            FibUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build(), FibUtil.DEFAULT_CALLBACK);
+        }
+        return false;
     }
 
     private void makeTunnelTableEntry(BigInteger dpId, long label, long groupId/*String egressInterfaceName*/,
                                       WriteTransaction tx) {
-        List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
-        actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
+      List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
+      actionsInfos.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
 
 
-        createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
+      createTerminatingServiceActions(dpId, (int)label, actionsInfos, tx);
 
-        LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
-                dpId, label, groupId);
-    }
+      LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully",
+              dpId, label, groupId);
+  }
 
-    public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
-                                                 WriteTransaction tx) {
-        List<MatchInfo> mkMatches = new ArrayList<>();
+  public void createTerminatingServiceActions( BigInteger destDpId, int label, List<ActionInfo> actionsInfos,
+                                               WriteTransaction tx) {
+      List<MatchInfo> mkMatches = new ArrayList<>();
 
-        LOG.info("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
+      LOG.info("create terminatingServiceAction on DpnId = {} and serviceId = {} and actions = {}", destDpId , label,actionsInfos);
 
-        // Matching metadata
-        // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
-        mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+      // Matching metadata
+      // FIXME vxlan vni bit set is not working properly with OVS.need to revisit
+      mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
 
-        List<InstructionInfo> mkInstructions = new ArrayList<>();
-        mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
+      List<InstructionInfo> mkInstructions = new ArrayList<>();
+      mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos));
 
-        FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
-                getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
-                0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
+      FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,
+                      getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label),
+                      0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)),mkMatches, mkInstructions);
 
-        FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
+      FlowKey flowKey = new FlowKey( new FlowId(terminatingServiceTableFlowEntity.getFlowId()) );
 
-        FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
+      FlowBuilder flowbld = terminatingServiceTableFlowEntity.getFlowBuilder();
 
-        Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
-        InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
-        tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
   }
+      Node nodeDpn = buildDpnNode(terminatingServiceTableFlowEntity.getDpnId());
+      InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
+              .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+              .child(Table.class, new TableKey(terminatingServiceTableFlowEntity.getTableId())).child(Flow.class,flowKey).build();
+      tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flowbld.build(),true );
+ }
 
-    private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
-        FlowEntity flowEntity;
-        LOG.info("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
-        List<MatchInfo> mkMatches = new ArrayList<>();
-        // Matching metadata
-        mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
-        flowEntity = MDSALUtil.buildFlowEntity(dpId,
-                NwConstants.INTERNAL_TUNNEL_TABLE,
-                getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
-                5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
-                COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
-        Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
-        FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
-        InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
+  private void removeTunnelTableEntry(BigInteger dpId, long label, WriteTransaction tx) {
+    FlowEntity flowEntity;
+    LOG.info("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId , label);
+    List<MatchInfo> mkMatches = new ArrayList<>();
+    // Matching metadata
+    mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)}));
+    flowEntity = MDSALUtil.buildFlowEntity(dpId,
+                                           NwConstants.INTERNAL_TUNNEL_TABLE,
+                                           getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int)label),
+                                           5, String.format("%s:%d","TST Flow Entry ",label), 0, 0,
+                                           COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null);
+    Node nodeDpn = buildDpnNode(flowEntity.getDpnId());
+    FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId()));
+    InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
+      .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+      .child(Table.class, new TableKey(flowEntity.getTableId())).child(Flow.class, flowKey).build();
 
-        tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
-        LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
-    }
+    tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
+    LOG.debug("Terminating service Entry for dpID {} : label : {} removed successfully", dpId, label);
+  }
 
     /**
      * Delete local FIB entry
@@ -752,39 +879,59 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
      * @param vrfEntry
      * @return
      */
-    public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
-        List<BigInteger> returnLocalDpnId = new ArrayList<>();
-        BigInteger localDpnId = BigInteger.ZERO;
-        VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
-        String localNextHopIP = vrfEntry.getDestPrefix();
-
-        if (localNextHopInfo == null) {
-            //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
-            Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            if (extra_route != null) {
-                //FIXME:Suraj
-//                for (String nextHopIp : extra_route.getNexthopIpList()) {
-//                    LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
-//                    if (nextHopIp != null) {
-//                        localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
-//                        localNextHopIP = nextHopIp + "/32";
-//                        BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, localDpnId,
-//                                vpnId, rd, vrfEntry, true /*isExtraRoute*/);
-//                        returnLocalDpnId.add(dpnId);
-//                    }
-//                }
-            }
-        } else {
-            BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP, localDpnId,
-                    vpnId, rd, vrfEntry, false /*isExtraRoute*/);
-            returnLocalDpnId.add(dpnId);
-        }
-
-        return returnLocalDpnId;
-    }
+  public List<BigInteger> deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) {
+      List<BigInteger> returnLocalDpnId = new ArrayList<>();
+      BigInteger localDpnId = BigInteger.ZERO;
+      VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
+      String localNextHopIP = vrfEntry.getDestPrefix();
+
+      if (localNextHopInfo == null) {
+          //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+          Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+          if (extra_route != null) {
+              for (String nextHopIp : extra_route.getNexthopIpList()) {
+                  LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
+                  if (nextHopIp != null) {
+                      localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp + "/32");
+                      localNextHopIP = nextHopIp + "/32";
+                      BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
+                                                                  vpnId, rd, vrfEntry, true /*isExtraRoute*/);
+                      if (!dpnId.equals(BigInteger.ZERO)) {
+                          returnLocalDpnId.add(dpnId);
+                      }
+                  }
+              }
+          }
+
+          if (localNextHopInfo == null) {
+              /* Imported VRF entry */
+              LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+              if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
+                      vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+                  VpnNexthopBuilder vpnNexthopBuilder = new VpnNexthopBuilder();
+                  vpnNexthopBuilder.setDpnId(lri.getDpnId());
+                  BigInteger dpnId = checkDeleteLocalFibEntry(vpnNexthopBuilder.build(), localNextHopIP,
+                          vpnId, rd, vrfEntry, false /*isExtraRoute*/);
+                  if (!dpnId.equals(BigInteger.ZERO)) {
+                      returnLocalDpnId.add(dpnId);
+                  }
+              }
+          }
+
+
+      } else {
+          BigInteger dpnId = checkDeleteLocalFibEntry(localNextHopInfo, localNextHopIP,
+                                                      vpnId, rd, vrfEntry, false /*isExtraRoute*/);
+          if (!dpnId.equals(BigInteger.ZERO)) {
+              returnLocalDpnId.add(dpnId);
+          }
+      }
+
+      return returnLocalDpnId;
+  }
 
     private BigInteger checkDeleteLocalFibEntry(VpnNexthop localNextHopInfo, final String localNextHopIP,
-                                                final BigInteger localDpnId, final Long vpnId, final String rd,
+                                                final Long vpnId, final String rd,
                                                 final VrfEntry vrfEntry, final boolean isExtraRoute) {
         if (localNextHopInfo != null) {
             final BigInteger dpnId = localNextHopInfo.getDpnId();;
@@ -794,35 +941,36 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                         @Override
                         public List<ListenableFuture<Void>> call() throws Exception {
                             WriteTransaction tx = broker.newWriteOnlyTransaction();
-
-                            Prefixes prefix = getPrefixToInterface(vpnId, isExtraRoute ? localNextHopIP : vrfEntry.getDestPrefix());
                             makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null /* instructions */,
                                     NwConstants.DEL_FLOW, tx);
-                            makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
-                                    DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
-                            removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
+                            if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                                makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null /* instructions */,
+                                        DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
+                                removeTunnelTableEntry(dpnId, vrfEntry.getLabel(), tx);
+                            }
                             List<ListenableFuture<Void>> futures = new ArrayList<>();
                             futures.add(tx.submit());
                             return futures;
                         }
                     });
             //TODO: verify below adjacency call need to be optimized (?)
-            deleteLocalAdjacency(localDpnId, vpnId, localNextHopIP);
+            deleteLocalAdjacency(dpnId, vpnId, localNextHopIP);
+            return dpnId;
         }
-        return localDpnId;
+        return BigInteger.ZERO;
     }
 
 
-    private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
-        return InstanceIdentifier.builder(PrefixToInterface.class)
-                .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
-    }
+  private InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) {
+    return InstanceIdentifier.builder(PrefixToInterface.class)
+        .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build();
+  }
 
-    private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
-        Optional<Prefixes> localNextHopInfoData =
-                FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
-        return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
-    }
+  private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) {
+    Optional<Prefixes> localNextHopInfoData =
+        FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix));
+    return  localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null;
+  }
 
     private InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
         return InstanceIdentifier.builder(VpnToExtraroute.class)
@@ -837,24 +985,24 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
     }
 
-    private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
+  private Class<? extends TunnelTypeBase> getTunnelType(String ifName) {
         try {
             Future<RpcResult<GetTunnelTypeOutput>> result = interfaceManager.getTunnelType(
-                    new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
-            RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
-            if(!rpcResult.isSuccessful()) {
-                LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
-            } else {
-                return rpcResult.getResult().getTunnelType();
-            }
+                       new GetTunnelTypeInputBuilder().setIntfName(ifName).build());
+          RpcResult<GetTunnelTypeOutput> rpcResult = result.get();
+          if(!rpcResult.isSuccessful()) {
+              LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
+          } else {
+              return rpcResult.getResult().getTunnelType();
+          }
 
-        } catch (InterruptedException | ExecutionException e) {
-            LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
-        }
+      } catch (InterruptedException | ExecutionException e) {
+          LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e);
+      }
 
-        return null;
+  return null;
 
-    }
+  }
     private void createRemoteFibEntry(final BigInteger remoteDpnId, final long vpnId, final VrfTablesKey vrfTableKey,
                                       final VrfEntry vrfEntry, WriteTransaction tx) {
         Boolean wrTxPresent = true;
@@ -863,7 +1011,8 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
             tx = broker.newWriteOnlyTransaction();
         }
         String rd = vrfTableKey.getRouteDistinguisher();
-        LOG.debug("createremotefibentry: adding route {} for rd {} with transaction ", vrfEntry.getDestPrefix(), rd);
+        LOG.debug(  "createremotefibentry: adding route {} for rd {} with transaction {}",
+                    vrfEntry.getDestPrefix(), rd, tx);
         /********************************************/
         List<String> tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
 
@@ -915,28 +1064,25 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         InstanceIdentifier<VpnToDpnList> id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId);
         Optional<VpnToDpnList> dpnInVpn = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
         if (dpnInVpn.isPresent()) {
-            List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                    .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
-            org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
-                    currVpnInterface = new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder().setInterfaceName(intfName).build();
+            List<VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
+            VpnInterfaces currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 
             if (vpnInterfaces.remove(currVpnInterface)) {
                 if (vpnInterfaces.isEmpty()) {
-                    LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
-                    FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id);
-                    cleanUpDpnForVpn(dpnId, vpnId, rd);
+                  LOG.trace("Last vpn interface {} on dpn {} for vpn {}. Clean up fib in dpn", intfName, dpnId, rd);
+                  FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id);
+                  cleanUpDpnForVpn(dpnId, vpnId, rd);
                 } else {
-                    LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
+                  LOG.trace("Delete vpn interface {} from dpn {} to vpn {} list.", intfName, dpnId, rd);
                     FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
-                            org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                                    .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
-                            new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey(intfName)));
+                            VpnInterfaces.class,
+                            new VpnInterfacesKey(intfName)));
                 }
             }
         }
     }
 
-    private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
+  private void cleanUpOpDataForFib(Long vpnId, String rd, final VrfEntry vrfEntry) {
     /* Get interface info from prefix to interface mapping;
         Use the interface info to get the corresponding vpn interface op DS entry,
         remove the adjacency corresponding to this fib entry.
@@ -945,124 +1091,126 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
          - prefix to interface entry
          - vpn interface op DS
      */
-        LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
-        Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
-        Extraroute extraRoute = null;
-        if (prefixInfo == null) {
-            extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            if(extraRoute != null) {
-                //FIXME:Suraj
-//                for (String nextHopIp : extraRoute.getNexthopIpList()) {
-  //                  LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
-//
-  //                  if (nextHopIp != null) {
-    //                    prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
-      //                  checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
-        //            }
-          //      }
-            }
-        } else {
-            checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
-        }
-    }
-
-    private void checkCleanUpOpDataForFib(Prefixes prefixInfo, Long vpnId, String rd,
-                                          final VrfEntry vrfEntry, Extraroute extraRoute) {
-
-        if (prefixInfo == null) {
-            LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
-            return; //Don't have any info for this prefix (shouldn't happen); need to return
-        }
-
-        String ifName = prefixInfo.getVpnInterfaceName();
-        synchronized (ifName.intern()) {
-            Optional<VpnInterface> optvpnInterface = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
-                    FibUtil.getVpnInterfaceIdentifier(ifName));
-            if (optvpnInterface.isPresent()) {
-                long associatedVpnId = FibUtil.getVpnId(broker, optvpnInterface.get().getVpnInstanceName());
-                if (vpnId != associatedVpnId) {
-                    LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
-                            vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
-                    LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
-                    FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
-                            FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
-                    LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
-                    return;
-                } else {
-                    LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
+      LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId);
+      Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix());
+      Extraroute extraRoute = null;
+      if (prefixInfo == null) {
+          extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+          if(extraRoute != null) {
+              for (String nextHopIp : extraRoute.getNexthopIpList()) {
+                  LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp);
+
+                  if (nextHopIp != null) {
+                      prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32");
+                      checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
+                  }
+              }
+          }
+          if (prefixInfo == null) {
+              LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+              if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
+                      vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+                  PrefixesBuilder prefixBuilder = new PrefixesBuilder();
+                  prefixBuilder.setDpnId(lri.getDpnId());
+                  prefixBuilder.setVpnInterfaceName(lri.getVpnInterfaceName());
+                  prefixBuilder.setIpAddress(lri.getPrefix());
+                  prefixInfo = prefixBuilder.build();
+                  LOG.debug("Fetched labelRouteInfo for label {} interface {} and got dpn {}",
+                          vrfEntry.getLabel(), prefixInfo.getVpnInterfaceName(), lri.getDpnId());
+                  checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
+              }
+          }
+      } else {
+          checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute);
+      }
+  }
+
+  private void checkCleanUpOpDataForFib(Prefixes prefixInfo, Long vpnId, String rd,
+                                        final VrfEntry vrfEntry, Extraroute extraRoute) {
+
+      if (prefixInfo == null) {
+          LOG.debug("Cleanup VPN Data Failed as unable to find prefix Info for prefix {}", vrfEntry.getDestPrefix());
+          return; //Don't have any info for this prefix (shouldn't happen); need to return
+      }
+
+      String ifName = prefixInfo.getVpnInterfaceName();
+      synchronized (ifName.intern()) {
+          Optional<VpnInterface> optvpnInterface = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                                                                FibUtil.getVpnInterfaceIdentifier(ifName));
+          if (optvpnInterface.isPresent()) {
+              long associatedVpnId = FibUtil.getVpnId(broker, optvpnInterface.get().getVpnInstanceName());
+              if (vpnId != associatedVpnId) {
+                  LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}",
+                          vrfEntry.getDestPrefix(), associatedVpnId, vpnId);
+                  LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix());
+                  FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
+                          FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
+                  LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix());
+                  return;
+              } else {
+                  LOG.debug("Processing cleanup of prefix {} associated with vpn {}",
                             vrfEntry.getDestPrefix(), associatedVpnId);
-                }
-            }
-            if (extraRoute != null) {
-                FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
-                        FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
-            }
-            Optional<Adjacencies> optAdjacencies = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
-                    FibUtil.getAdjListPath(ifName));
-            int numAdj = 0;
-            if (optAdjacencies.isPresent()) {
-                numAdj = optAdjacencies.get().getAdjacency().size();
-            }
-            //remove adjacency corr to prefix
-            if (numAdj > 1) {
-                LOG.trace("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
-                FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
-                        FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
-            }
-            if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
-                //clean up the vpn interface from DpnToVpn list
-                LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
-                FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
-                        FibUtil.getVpnInterfaceIdentifier(ifName));
-            }
-            FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
-                    FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
-        }
-    }
-
-
-    private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
-        final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
-
-        final String rd  = vrfTableKey.getRouteDistinguisher();
-        final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
-        if (vpnInstance == null) {
-            LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
-            return;
-        }
-        final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
-        SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
-        if (subnetRoute != null) {
-            if (vpnToDpnList != null) {
-                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
-                        new Callable<List<ListenableFuture<Void>>>() {
-                            @Override
-                            public List<ListenableFuture<Void>> call() throws Exception {
-                                WriteTransaction tx = broker.newWriteOnlyTransaction();
-
-                                for (final VpnToDpnList curDpn : vpnToDpnList) {
-
-                                    makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
-                                            vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
-                                    makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
-                                            DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
-                                }
-                                List<ListenableFuture<Void>> futures = new ArrayList<>();
-                                futures.add(tx.submit());
-                                return futures;
-                            }
-                        });
-            }
-            FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
-                    FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
-            LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
-                    vrfEntry.getDestPrefix());
-            return;
-        }
-
-        final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
-                vrfTableKey.getRouteDistinguisher(), vrfEntry);
+              }
+          }
+          if (extraRoute != null) {
+              FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
+                             FibUtil.getVpnToExtrarouteIdentifier(rd, vrfEntry.getDestPrefix()));
+          }
+          Optional<Adjacencies> optAdjacencies = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                                                              FibUtil.getAdjListPath(ifName));
+          int numAdj = 0;
+          if (optAdjacencies.isPresent()) {
+              numAdj = optAdjacencies.get().getAdjacency().size();
+          }
+          //remove adjacency corr to prefix
+          if (numAdj > 1) {
+              LOG.trace("cleanUpOpDataForFib: remove adjacency for prefix: {} {}", vpnId, vrfEntry.getDestPrefix());
+              FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
+                             FibUtil.getAdjacencyIdentifier(ifName, vrfEntry.getDestPrefix()));
+          }
+          if ((numAdj - 1) == 0) { //there are no adjacencies left for this vpn interface, clean up
+              //clean up the vpn interface from DpnToVpn list
+              LOG.trace("Clean up vpn interface {} from dpn {} to vpn {} list.", ifName, prefixInfo.getDpnId(), rd);
+              FibUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
+                             FibUtil.getVpnInterfaceIdentifier(ifName));
+          }
+
+          synchronized (vrfEntry.getLabel().toString().intern()) {
+              LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+              if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) &&
+                      vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+                  Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
+                  String vpnInstanceName = "";
+                  if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                      vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                  }
+                  boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
+                  if (lriRemoved) {
+                      String parentRd = lri.getParentVpnRd();
+                      FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
+                              FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
+                  }
+              }
+          }
+      }
+  }
+
+  private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
+    final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
+
+    final String rd  = vrfTableKey.getRouteDistinguisher();
+    final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(vrfTableKey.getRouteDistinguisher());
+    if (vpnInstance == null) {
+        LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
+        return;
+    }
+    final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
+    long elanTag = 0L;
+    SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+    if (subnetRoute != null) {
+        elanTag = subnetRoute.getElantag();
+        LOG.trace("SubnetRoute augmented vrfentry found for rd {} prefix {} with elantag {}",
+                rd, vrfEntry.getDestPrefix(), elanTag);
         if (vpnToDpnList != null) {
             DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
             dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
@@ -1071,17 +1219,13 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                         public List<ListenableFuture<Void>> call() throws Exception {
                             WriteTransaction tx = broker.newWriteOnlyTransaction();
 
-                            if (localDpnIdList.size() <= 0) {
-                                for (VpnToDpnList curDpn : vpnToDpnList) {
-                                    deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                }
-                            } else {
-                                for (BigInteger localDpnId : localDpnIdList) {
-                                    for (VpnToDpnList curDpn : vpnToDpnList) {
-                                        if (!curDpn.getDpnId().equals(localDpnId)) {
-                                            deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
-                                        }
-                                    }
+                            for (final VpnToDpnList curDpn : vpnToDpnList) {
+
+                                makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry,
+                                        vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx);
+                                if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
+                                    makeLFibTableEntry(curDpn.getDpnId(), vrfEntry.getLabel(), null,
+                                            DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
                                 }
                             }
                             List<ListenableFuture<Void>> futures = new ArrayList<>();
@@ -1090,68 +1234,117 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                         }
                     });
         }
-
-        //The flow/group entry has been deleted from config DS; need to clean up associated operational
-        //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
-        cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
-
-        // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
-        // of the interVpnLink.
-        Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(broker, rd);
-        if ( vpnUuid.isPresent() ) {
-            Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(broker, vpnUuid.get());
-            String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
-
-            if ( interVpnLink.isPresent()
-                    && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
-                    && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
-                    || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
-                    && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
-                // This is route that points to the other endpoint of an InterVpnLink
-                // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
-                removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
+        synchronized (vrfEntry.getLabel().toString().intern()) {
+            LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+            if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+                Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(broker, rd);
+                String vpnInstanceName = "";
+                if (vpnInstanceOpDataEntryOptional.isPresent()) {
+                    vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
+                }
+                boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName);
+                if (lriRemoved) {
+                    String parentRd = lri.getParentVpnRd();
+                    FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME,
+                            FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
+                    LOG.trace("deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
+                            vrfEntry.getDestPrefix());
+                }
             }
         }
-
+        return;
     }
 
-    public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
-                                  final long vpnId, final VrfTablesKey vrfTableKey,
-                                  final VrfEntry vrfEntry, WriteTransaction tx) {
+    final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(),
+                                                          vrfTableKey.getRouteDistinguisher(), vrfEntry);
+    if (vpnToDpnList != null) {
+        DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+        dataStoreCoordinator.enqueueJob("FIB" + rd.toString() + vrfEntry.getDestPrefix(),
+                new Callable<List<ListenableFuture<Void>>>() {
+                    @Override
+                    public List<ListenableFuture<Void>> call() throws Exception {
+                        WriteTransaction tx = broker.newWriteOnlyTransaction();
 
-        Boolean wrTxPresent = true;
-        if (tx == null) {
-            wrTxPresent = false;
-            tx = broker.newWriteOnlyTransaction();
-        }
-
-        LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
-        String rd = vrfTableKey.getRouteDistinguisher();
-
-        if(localDpnId != null) {
-            // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
-            deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
-            return;
-        }
-
-        // below two reads are kept as is, until best way is found to identify dpnID
-        VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
-        Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-
-        if (localNextHopInfo == null && extraRoute != null) {
-            // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
-            //FIXME:Suraj
-//            for (String nextHopIp : extraRoute.getNexthopIpList()) {
-  //              localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
-    //            checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
-      //      }
-        } else {
-            checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
-        }
-        if(!wrTxPresent ){
-            tx.submit();
-        }
-    }
+                        if (localDpnIdList.size() <= 0) {
+                            for (VpnToDpnList curDpn : vpnToDpnList) {
+                                deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
+                            }
+                        } else {
+                            for (BigInteger localDpnId : localDpnIdList) {
+                                for (VpnToDpnList curDpn : vpnToDpnList) {
+                                    if (!curDpn.getDpnId().equals(localDpnId)) {
+                                        deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, tx);
+                                    }
+                                }
+                            }
+                        }
+                        List<ListenableFuture<Void>> futures = new ArrayList<>();
+                        futures.add(tx.submit());
+                        return futures;
+                    }
+                });
+    }
+
+    //The flow/group entry has been deleted from config DS; need to clean up associated operational
+    //DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
+    cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
+
+    // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
+    // of the interVpnLink.
+    Optional<String> vpnUuid = FibUtil.getVpnNameFromRd(broker, rd);
+    if ( vpnUuid.isPresent() ) {
+        Optional<InterVpnLink> interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(broker, vpnUuid.get());
+        String routeNexthop = vrfEntry.getNextHopAddressList().get(0);
+
+        if ( interVpnLink.isPresent()
+             && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get())
+                   && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop))
+                  || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() )
+                      && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) {
+            // This is route that points to the other endpoint of an InterVpnLink
+            // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it.
+            removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry);
+        }
+    }
+
+  }
+
+  public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
+                                final long vpnId, final VrfTablesKey vrfTableKey,
+                                final VrfEntry vrfEntry, WriteTransaction tx) {
+
+      Boolean wrTxPresent = true;
+      if (tx == null) {
+          wrTxPresent = false;
+          tx = broker.newWriteOnlyTransaction();
+      }
+
+    LOG.debug("deleting route: prefix={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
+    String rd = vrfTableKey.getRouteDistinguisher();
+
+      if(localDpnId != null) {
+          // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
+          deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx);
+          return;
+      }
+
+      // below two reads are kept as is, until best way is found to identify dpnID
+      VpnNexthop localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
+      Extraroute extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+
+      if (localNextHopInfo == null && extraRoute != null) {
+          // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
+          for (String nextHopIp : extraRoute.getNexthopIpList()) {
+              localNextHopInfo = nextHopManager.getVpnNexthop(vpnId, nextHopIp);
+              checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
+          }
+      } else {
+              checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx);
+      }
+      if(!wrTxPresent ){
+          tx.submit();
+      }
+  }
 
     private boolean checkDpnDeleteFibEntry(VpnNexthop localNextHopInfo, BigInteger remoteDpnId, long vpnId,
                                            VrfEntry vrfEntry, String rd, WriteTransaction tx){
@@ -1169,76 +1362,76 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
     }
 
     private void deleteFibEntry(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry, String rd, WriteTransaction tx){
-        makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
-        LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
-    }
-
-    private long get
-            (byte[] rawIpAddress) {
-        return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
-                + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
+            makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
+            LOG.debug("Successfully delete FIB entry: vrfEntry={}, vpnId={}", vrfEntry.getDestPrefix(), vpnId);
+    }
+
+  private long get
+          (byte[] rawIpAddress) {
+    return (((rawIpAddress[0] & 0xFF) << (3 * 8)) + ((rawIpAddress[1] & 0xFF) << (2 * 8))
+            + ((rawIpAddress[2] & 0xFF) << (1 * 8)) + (rawIpAddress[3] & 0xFF)) & 0xffffffffL;
+  }
+
+  private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
+                                  List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
+    Boolean wrTxPresent = true;
+    if (tx == null) {
+      wrTxPresent = false;
+      tx = broker.newWriteOnlyTransaction();
+    }
+
+    LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
+    String values[] = vrfEntry.getDestPrefix().split("/");
+    String ipAddress = values[0];
+    int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
+    if (addOrRemove == NwConstants.ADD_FLOW) {
+        LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
+    } else {
+        LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
+    }
+    InetAddress destPrefix;
+    try {
+      destPrefix = InetAddress.getByName(ipAddress);
+    } catch (UnknownHostException e) {
+      LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
+      return;
+    }
+
+    List<MatchInfo> matches = new ArrayList<>();
+
+    matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
+        BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
+
+    matches.add(new MatchInfo(MatchFieldType.eth_type,
+                              new long[] { NwConstants.ETHTYPE_IPV4 }));
+
+    if(prefixLength != 0) {
+      matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
+          destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
+    }
+    int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
+    String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
+    FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
+                                                      COOKIE_VM_FIB_TABLE, matches, instructions);
+
+    Flow flow = flowEntity.getFlowBuilder().build();
+    String flowId = flowEntity.getFlowId();
+    FlowKey flowKey = new FlowKey( new FlowId(flowId));
+    Node nodeDpn = buildDpnNode(dpId);
+
+    InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
+            .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
+            .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
+    if (addOrRemove == NwConstants.ADD_FLOW) {
+        tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
+    } else {
+        tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
     }
 
-    private void makeConnectedRoute(BigInteger dpId, long vpnId, VrfEntry vrfEntry, String rd,
-                                    List<InstructionInfo> instructions, int addOrRemove, WriteTransaction tx) {
-        Boolean wrTxPresent = true;
-        if (tx == null) {
-            wrTxPresent = false;
-            tx = broker.newWriteOnlyTransaction();
-        }
-
-        LOG.trace("makeConnectedRoute: vrfEntry {}", vrfEntry);
-        String values[] = vrfEntry.getDestPrefix().split("/");
-        String ipAddress = values[0];
-        int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]);
-        if (addOrRemove == NwConstants.ADD_FLOW) {
-            LOG.debug("Adding route to DPN {} for rd {} prefix {} ", dpId, rd, vrfEntry.getDestPrefix());
-        } else {
-            LOG.debug("Removing route from DPN {} for rd {} prefix {}", dpId, rd, vrfEntry.getDestPrefix());
-        }
-        InetAddress destPrefix;
-        try {
-            destPrefix = InetAddress.getByName(ipAddress);
-        } catch (UnknownHostException e) {
-            LOG.error("Failed to get destPrefix for prefix {} ", vrfEntry.getDestPrefix(), e);
-            return;
-        }
-
-        List<MatchInfo> matches = new ArrayList<>();
-
-        matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
-                BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID }));
-
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_IPV4 }));
-
-        if(prefixLength != 0) {
-            matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] {
-                    destPrefix.getHostAddress(), Integer.toString(prefixLength)}));
-        }
-        int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength;
-        String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix);
-        FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0,
-                COOKIE_VM_FIB_TABLE, matches, instructions);
-
-        Flow flow = flowEntity.getFlowBuilder().build();
-        String flowId = flowEntity.getFlowId();
-        FlowKey flowKey = new FlowKey( new FlowId(flowId));
-        Node nodeDpn = buildDpnNode(dpId);
-
-        InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class)
-                .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class)
-                .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
-        if (addOrRemove == NwConstants.ADD_FLOW) {
-            tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true);
-        } else {
-            tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
-        }
-
-        if(!wrTxPresent ){
-            tx.submit();
-        }
+    if(!wrTxPresent ){
+      tx.submit();
     }
+  }
 
     //TODO: How to handle the below code, its a copy paste from MDSALManager.java
     private Node buildDpnNode(BigInteger dpnId) {
@@ -1258,7 +1451,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         List<MatchInfo> matches = new ArrayList<MatchInfo>();
         matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_MPLS_UC }));
+                                  new long[] { NwConstants.ETHTYPE_MPLS_UC }));
         matches.add(new MatchInfo(MatchFieldType.mpls_label, new String[]{Long.toString(label)}));
 
         // Install the flow entry in L3_LFIB_TABLE
@@ -1266,7 +1459,7 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
 
         FlowEntity flowEntity;
         flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_LFIB_TABLE, flowRef, priority, flowRef, 0, 0,
-                COOKIE_VM_LFIB_TABLE, matches, instructions);
+                                               COOKIE_VM_LFIB_TABLE, matches, instructions);
         Flow flow = flowEntity.getFlowBuilder().build();
         String flowId = flowEntity.getFlowId();
         FlowKey flowKey = new FlowKey( new FlowId(flowId));
@@ -1286,216 +1479,340 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
         LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}",dpId, label, instructions );
     }
 
-    private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
-        LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
-        try {
-            nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
-        } catch (NullPointerException e) {
-            LOG.trace("", e);
-        }
-    }
-
-    public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd) {
-        LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
-        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
-        String lockOnDpnVpn = new String(dpnId.toString() + vpnId);
-        synchronized (lockOnDpnVpn.intern()) {
-            final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
-            if (vrfTable.isPresent()) {
-                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB" + vpnId + dpnId.toString(),
-                        new Callable<List<ListenableFuture<Void>>>() {
-                            @Override
-                            public List<ListenableFuture<Void>> call() throws Exception {
-                                WriteTransaction tx = broker.newWriteOnlyTransaction();
-                                for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-
-                                    SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
-                                    if (subnetRoute != null) {
-                                        long elanTag = subnetRoute.getElantag();
-                                        installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
-                                        continue;
-                                    }
-                                    // Passing null as we don't know the dpn
-                                    // to which prefix is attached at this point
-                                    createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
-                                }
-                                //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
-                                List<ListenableFuture<Void>> futures = new ArrayList<>();
-                                futures.add(tx.submit());
-                                return futures;
-                            }
-                        });
-            }
-        }
-    }
-
-    public void populateFibOnDpn(BigInteger dpnId, BigInteger localDpnId, long vpnId, String rd, String nexthopIp) {
-        LOG.trace(" dpn {} for vpn {}, rd {}, nexthopIp {} : populateFibOnDpn", dpnId, vpnId, rd, nexthopIp);
-        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
-        String lockOnDpnVpn = new String(dpnId.toString()+ vpnId);
-        synchronized (lockOnDpnVpn.intern()) {
-            Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
-            if (vrfTable.isPresent()) {
-                for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                    // Passing null as we don't know the dpn
-                    // to which prefix is attached at this point
-                    if (nexthopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                        LOG.trace(" creating remote FIB entry");
-                        createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, null);
-                    }
-                }
-            }
-        }
-    }
-
-    public void handleRemoteRoute(boolean action, BigInteger localDpnId, BigInteger remoteDpnId, long vpnId, String  rd, String destPrefix ,
-                                  String nextHopIp) {
-        VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
-        VrfEntry vrfEntry = getVrfEntry(broker, rd, destPrefix);
-        if (vrfEntry == null) {
-            LOG.trace("VrfEntry for prefix {} is null. Exitting.", nextHopIp);
-            return;
-        }
-        LOG.trace("handleRemoteRoute --- action {}, localDpnId {}, remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
-                action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
-        if (action == true) {
-            createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, null);
-        } else {
-            deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, null);
-        }
-
-    }
-
-    public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd) {
-        LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
-        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
-        String lockOnDpnVpn = new String(dpnId.toString() + vpnId);
-        synchronized (lockOnDpnVpn.intern()) {
-
-            final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
-            if (vrfTable.isPresent()) {
-                DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
-                dataStoreCoordinator.enqueueJob("FIB" + vpnId + dpnId.toString(),
-                        new Callable<List<ListenableFuture<Void>>>() {
-                            WriteTransaction tx = broker.newWriteOnlyTransaction();
-                            @Override
-                            public List<ListenableFuture<Void>> call() throws Exception {
-                                for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                            /* Handle subnet routes here */
-                                    SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
-                                    if (subnetRoute != null) {
-                                        LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
-                                                dpnId, rd);
-                                        makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
-                                        makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
-                                        LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
+  private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) {
+    LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress);
+    try {
+      nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress);
+    } catch (NullPointerException e) {
+      LOG.trace("", e);
+    }
+  }
+
+  public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd) {
+      LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd);
+      InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+      synchronized (rd.intern()) {
+          final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+          if (vrfTable.isPresent()) {
+              DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+              dataStoreCoordinator.enqueueJob("FIB" + vpnId + dpnId.toString(),
+                      new Callable<List<ListenableFuture<Void>>>() {
+                          @Override
+                          public List<ListenableFuture<Void>> call() throws Exception {
+                              WriteTransaction tx = broker.newWriteOnlyTransaction();
+                              for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+
+                                  SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+                                  if (subnetRoute != null) {
+                                      long elanTag = subnetRoute.getElantag();
+                                      installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx);
+                                      continue;
+                                  }
+                                  if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports
+                                      LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel());
+                                      if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) {
+                                          if (lri.getDpnId().equals(dpnId)) {
+                                              createLocalFibEntry(vpnId, rd, vrfEntry);
+                                              continue;
+                                          }
+                                      }
+                                  }
+                                  // Passing null as we don't know the dpn
+                                  // to which prefix is attached at this point
+                                  createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
+                              }
+                              //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?)
+                              List<ListenableFuture<Void>> futures = new ArrayList<>();
+                              futures.add(tx.submit());
+                              return futures;
+                          }
+                      });
+          }
+      }
+  }
+
+  public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd, final String localNextHopIp, final String remoteNextHopIp) {
+    LOG.trace(  "dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn",
+                dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+    InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+    synchronized (rd.intern()) {
+      final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+      if (vrfTable.isPresent()) {
+          DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+          dataStoreCoordinator.enqueueJob(" FIB + on Dpn , rd "
+                            + rd.toString() + "localNextHopIp "
+                            + localNextHopIp + "remoteNextHopIP"
+                            + remoteNextHopIp + "vpnId "
+                            + vpnId + "dpnId" + dpnId,
+                  new Callable<List<ListenableFuture<Void>>>() {
+                      @Override
+                      public List<ListenableFuture<Void>> call() throws Exception {
+                          WriteTransaction writeTransaction = broker.newWriteOnlyTransaction();
+                          List<ListenableFuture<Void>> futures = new ArrayList<>();
+                          LOG.trace("populate FIB starts on Dpn " + dpnId
+                                  + "rd  " + rd.toString()
+                                  + "localNextHopIp " + localNextHopIp
+                                  + "remoteNextHopIp" + remoteNextHopIp
+                                  + "vpnId " + vpnId );
+
+                          for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                                LOG.trace("old vrfEntry before populate:: {}", vrfEntry);
+
+                              if (vrfEntry.getOrigin().equals(RouteOrigin.BGP.getValue())) {
+                                  if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                      LOG.trace(" creating remote FIB entry for vfEntry {}", vrfEntry);
+                                      createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
+                                  }
+                              }
+
+                              if ((vrfEntry.getOrigin().equals(RouteOrigin.CONNECTED.getValue())) ||
+                                  (vrfEntry.getOrigin().equals(RouteOrigin.STATIC.getValue()))) {
+                                  String destPfx = vrfEntry.getDestPrefix();
+                                  BigInteger dpnIdForPrefix = nextHopManager.getDpnForPrefix(vpnId, destPfx);
+                                  if (dpnIdForPrefix == null) {
+                                      LOG.trace("Populate::the dpnIdForPrefix is null for prefix {}.",
                                                 vrfEntry.getDestPrefix());
-                                        continue;
-                                    }
-                                    // Passing null as we don't know the dpn
-                                    // to which prefix is attached at this point
-                                    deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
-                                }
-                                List<ListenableFuture<Void>> futures = new ArrayList<>();
-                                futures.add(tx.submit());
-                                return futures;
-                            }
-
-                        });
+                                      continue;
+                                  }
+                                  int sameDpnId = dpnIdForPrefix.compareTo(dpnId);
+                                  if (sameDpnId != 0) {
+                                      LOG.trace("Populate::Different srcDpnId {} and dpnIdForPrefix {} for prefix {}",
+                                              dpnId, dpnIdForPrefix, vrfEntry.getDestPrefix());
+                                     continue;
+                                  }
+
+                                  // Passing null as we don't know the dpn
+                                  // to which prefix is attached at this point
+                                  InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, vrfEntry.getDestPrefix());
+
+
+                                  vrfEntry.getNextHopAddressList().add(localNextHopIp);
+                                  VrfEntry newVrfEntry =
+                                          new VrfEntryBuilder(vrfEntry).setNextHopAddressList(vrfEntry.getNextHopAddressList()).build();
+                                  // Just update the VrfEntry
+                                  FibUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION,
+                                          vrfEntryId, newVrfEntry);
+                                  vrfEntry = getVrfEntry(broker, rd, destPfx);
+                                  LOG.trace("updated vrfEntry after populate:: {}", vrfEntry);
+                              }
+                          }
+                          futures.add(writeTransaction.submit());
+                          LOG.trace("populate FIB ends on Dpn " + dpnId
+                                  + "rd  " + rd.toString()
+                                  + "localNextHopIp " + localNextHopIp
+                                  + "remoteNextHopIp" + remoteNextHopIp
+                                  + "vpnId " + vpnId );
+                          return futures;
+                      }
+                  });
+      }
+    }
+  }
+
+  public void handleRemoteRoute(final boolean action, final BigInteger localDpnId, final BigInteger remoteDpnId, final long vpnId, final String  rd, final String destPrefix , final String localNextHopIP,
+                                final String remoteNextHopIp) {
+
+      DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+      dataStoreCoordinator.enqueueJob(  "FIB" + rd.toString()
+                                        + "local dpid" + localDpnId
+                                        + "remote dpid" + remoteDpnId
+                                        + "vpnId" + vpnId
+                                        + "localNHIp" + localNextHopIP
+                                        + "remoteNHIp" + remoteNextHopIp,
+              new Callable<List<ListenableFuture<Void>>>() {
+                  @Override
+                  public List<ListenableFuture<Void>> call() throws Exception {
+                      List<ListenableFuture<Void>> futures = new ArrayList<>();
+                      WriteTransaction writeTransaction = broker.newWriteOnlyTransaction();
+                      VrfTablesKey vrfTablesKey = new VrfTablesKey(rd);
+                      VrfEntry vrfEntry = getVrfEntry(broker, rd, destPrefix);
+                      if (vrfEntry == null)
+                          return futures;
+                      LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " +
+                                "remoteDpnId {} , vpnId {}, rd {}, destPfx {}",
+                                action, localDpnId, remoteDpnId, vpnId, rd, destPrefix);
+                      if (action == true) {
+                          vrfEntry = getVrfEntry(broker, rd, destPrefix);
+                          LOG.trace("handleRemoteRoute updated(add)  vrfEntry :: {}", vrfEntry);
+                          createRemoteFibEntry(remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
+                      } else {
+                          vrfEntry = getVrfEntry(broker, rd, destPrefix);
+                          LOG.trace("handleRemoteRoute updated(remove)  vrfEntry :: {}", vrfEntry);
+                          deleteRemoteRoute(null, remoteDpnId, vpnId, vrfTablesKey, vrfEntry, writeTransaction);
+                      }
+                      futures.add(writeTransaction.submit());
+                      return futures;
+                  }
+              });
+  }
+
+  public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd) {
+      LOG.trace("Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
+      InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+      synchronized (rd.intern()) {
+          final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+          if (vrfTable.isPresent()) {
+              DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+              dataStoreCoordinator.enqueueJob("FIB" + vpnId + dpnId.toString(),
+                      new Callable<List<ListenableFuture<Void>>>() {
+                          WriteTransaction tx = broker.newWriteOnlyTransaction();
+                          @Override
+                          public List<ListenableFuture<Void>> call() throws Exception {
+                              for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                            /* Handle subnet routes here */
+                                  SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
+                                  if (subnetRoute != null) {
+                                      LOG.trace("Cleaning subnetroute {} on dpn {} for vpn {} : cleanUpDpnForVpn", vrfEntry.getDestPrefix(),
+                                              dpnId, rd);
+                                      makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx);
+                                      makeLFibTableEntry(dpnId, vrfEntry.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
+                                      LOG.trace("cleanUpDpnForVpn: Released subnetroute label {} for rd {} prefix {}", vrfEntry.getLabel(), rd,
+                                              vrfEntry.getDestPrefix());
+                                      continue;
+                                  }
+                                  // Passing null as we don't know the dpn
+                                  // to which prefix is attached at this point
+                                  deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx);
+                              }
+                              List<ListenableFuture<Void>> futures = new ArrayList<>();
+                              futures.add(tx.submit());
+                              return futures;
+                          }
+
+                      });
+          }
+
+      }
+  }
+
+  public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd,
+                               final String localNextHopIp, final String remoteNextHopIp) {
+    LOG.trace(  " cleanup remote routes on dpn {} for vpn {}, rd {}, " +
+                " localNexthopIp {} , remoteNexhtHopIp {} : cleanUpDpnForVpn",
+                dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp);
+    InstanceIdentifier<VrfTables> id = buildVrfId(rd);
+    synchronized (rd.intern()) {
+      final Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+      if (vrfTable.isPresent()) {
+          DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
+          dataStoreCoordinator.enqueueJob(" FIB + on Dpn " + dpnId
+                                            + rd + rd.toString()
+                                            + "localNextHopIp " + localNextHopIp
+                                            + "remoteNextHopIP" + remoteNextHopIp
+                                            + "vpnId " + vpnId
+                                            + "dpnId" + dpnId,
+                  new Callable<List<ListenableFuture<Void>>>() {
+                      @Override
+                      public List<ListenableFuture<Void>> call() throws Exception {
+                          WriteTransaction writeTransaction = broker.newWriteOnlyTransaction();
+                          List<ListenableFuture<Void>> futures = new ArrayList<>();
+                          LOG.trace("cleanup FIB starts on Dpn " + dpnId
+                                    + "rd  " + rd.toString()
+                                    + "localNextHopIp " + localNextHopIp
+                                    + "remoteNextHopIp" + remoteNextHopIp
+                                    + "vpnId " + vpnId );
+
+                          for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
+                              LOG.trace("old vrfEntry before cleanup:: {}", vrfEntry);
+                              if (remoteNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                  LOG.trace(" deleting remote FIB entry {}", vrfEntry);
+                                  deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, writeTransaction);
+                              }
+
+                              if (localNextHopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
+                                  LOG.trace("changing the nexthopip for local VM routes {} on dpn {}",
+                                            vrfEntry.getDestPrefix(), dpnId);
+                                  String destPfx = vrfEntry.getDestPrefix();
+                                  InstanceIdentifier<VrfEntry> vrfEntryId = getVrfEntryId(rd, destPfx);
+                                  List<java.lang.String> newList = vrfEntry.getNextHopAddressList();
+                                  newList.remove(localNextHopIp);
+                                 VrfEntry newVrfEntry =
+                                          new VrfEntryBuilder(vrfEntry).setNextHopAddressList(newList).build();
+                                  FibUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION,
+                                          vrfEntryId, newVrfEntry);
+                                  vrfEntry = getVrfEntry(broker, rd, destPfx);
+                                  LOG.trace("updated vrfEntry after cleanup:: {}", vrfEntry);
+                              }
+                          }
+                          futures.add(writeTransaction.submit());
+                          LOG.trace("cleanup FIB ends on Dpn " + dpnId
+                                    + "rd  " + rd.toString()
+                                    + "localNextHopIp " + localNextHopIp
+                                    + "remoteNextHopIp" + remoteNextHopIp
+                                    + "vpnId " + vpnId );
+                          return futures;
+                      }
+                  });
+
+      }
+    }
+  }
+
+  public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
+    InstanceIdentifierBuilder<VrfTables> idBuilder =
+        InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
+    InstanceIdentifier<VrfTables> id = idBuilder.build();
+    return id;
+  }
+
+  private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
+    return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
+        .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
+        .append(priority).toString();
+  }
+
+  private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
+    return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
+        .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
+        .append(rd).append(NwConstants.FLOWID_SEPARATOR)
+        .append(priority).append(NwConstants.FLOWID_SEPARATOR)
+        .append(destPrefix.getHostAddress()).toString();
+  }
+
+  private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
+      return new StringBuilder(64).append(FLOWID_PREFIX)
+                                  .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
+                                  .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
+                                  .append(nextHop).toString();
+  }
+
+  protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId, final VrfEntry vrfEntry,
+                                          String rd) {
+    List<String> adjacencyList = new ArrayList<>();
+    LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}", remoteDpnId, vpnId, vrfEntry);
+    try {
+        Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
+        List<String> prefixIpList;
+        if (extra_route == null) {
+            prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
+        } else {
+            prefixIpList = new ArrayList<>();
+            for (String extraRouteIp : extra_route.getNexthopIpList()) {
+                prefixIpList.add(extraRouteIp + "/32");
             }
-
         }
-    }
 
-    public void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd, String nexthopIp) {
-        LOG.trace(" cleanup remote routes on dpn {} for vpn {}, rd {}, nexthopIp {} : cleanUpDpnForVpn", dpnId, vpnId, rd, nexthopIp);
-        InstanceIdentifier<VrfTables> id = buildVrfId(rd);
-        String lockOnDpnVpn = new String(dpnId.toString()+ vpnId);
-        synchronized (lockOnDpnVpn.intern()) {
-            Optional<VrfTables> vrfTable = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
-            if (vrfTable.isPresent()) {
-                for (VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
-                    LOG.trace(":vrfEntry :: {}", vrfEntry);
-                    // Passing null as we don't know the dpn
-                    // to which prefix is attached at this point
-                    if (nexthopIp.trim().equals(vrfEntry.getNextHopAddressList().get(0).trim())) {
-                        LOG.trace(" deleting remote FIB entry {}", vrfEntry);
-                        deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, null);
-                    }
+        for (String prefixIp : prefixIpList) {
+            for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
+                LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
+                String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId, prefixIp, nextHopIp);
+                if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
+                    adjacencyList.add(adjacency);
                 }
-            } else {
-                LOG.trace("No vrfTable is present");
             }
         }
+    } catch (NullPointerException e) {
+      LOG.trace("", e);
     }
+    return adjacencyList;
+  }
 
-    public static InstanceIdentifier<VrfTables> buildVrfId(String rd) {
-        InstanceIdentifierBuilder<VrfTables> idBuilder =
-                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
-        InstanceIdentifier<VrfTables> id = idBuilder.build();
-        return id;
-    }
-
-    private String getFlowRef(BigInteger dpnId, short tableId, long label, int priority) {
-        return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(label).append(NwConstants.FLOWID_SEPARATOR)
-                .append(priority).toString();
-    }
-
-    private String getFlowRef(BigInteger dpnId, short tableId, String rd, int priority, InetAddress destPrefix) {
-        return new StringBuilder(64).append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
-                .append(rd).append(NwConstants.FLOWID_SEPARATOR)
-                .append(priority).append(NwConstants.FLOWID_SEPARATOR)
-                .append(destPrefix.getHostAddress()).toString();
-    }
-
-    private String getInterVpnFibFlowRef(String interVpnLinkName, String prefix, String nextHop ) {
-        return new StringBuilder(64).append(FLOWID_PREFIX)
-                .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
-                .append(prefix).append(NwConstants.FLOWID_SEPARATOR)
-                .append(nextHop).toString();
-    }
-
-    protected List<String> resolveAdjacency(final BigInteger remoteDpnId, final long vpnId, final VrfEntry vrfEntry,
-                                            String rd) {
-        List<String> adjacencyList = new ArrayList<>();
-        LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}", remoteDpnId, vpnId, vrfEntry);
-        try {
-            Extraroute extra_route = getVpnToExtraroute(rd, vrfEntry.getDestPrefix());
-            List<String> prefixIpList;
-            if (extra_route == null) {
-                prefixIpList = Arrays.asList(vrfEntry.getDestPrefix());
-            } else {
-                prefixIpList = new ArrayList<>();
-                //FIXME:Suraj
-//                for (String extraRouteIp : extra_route.getNexthopIpList()) {
-  //                  prefixIpList.add(extraRouteIp + "/32");
-    //            }
-            }
-
-            for (String prefixIp : prefixIpList) {
-                for (String nextHopIp : vrfEntry.getNextHopAddressList()) {
-                    LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp);
-                    String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId, prefixIp, nextHopIp);
-                    if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) {
-                        adjacencyList.add(adjacency);
-                    }
-                }
-            }
-        } catch (NullPointerException e) {
-            LOG.trace("", e);
-        }
-        return adjacencyList;
-    }
-
-    protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
-        InstanceIdentifier<VpnInstanceOpDataEntry> id = InstanceIdentifier.create(VpnInstanceOpData.class).child(
-                VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
-        Optional<VpnInstanceOpDataEntry> vpnInstanceOpData = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
-        return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
-    }
+  protected VpnInstanceOpDataEntry getVpnInstance(String rd) {
+    InstanceIdentifier<VpnInstanceOpDataEntry> id = InstanceIdentifier.create(VpnInstanceOpData.class).child(
+        VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd));
+    Optional<VpnInstanceOpDataEntry> vpnInstanceOpData = FibUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
+    return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null;
+  }
 
     public void processNodeAdd(BigInteger dpnId) {
         LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId);
@@ -1515,11 +1832,11 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                 NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
 
         FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE,
-                getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE,
-                        NwConstants.TABLE_MISS_FLOW),
-                NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow",
-                0, 0, COOKIE_VM_FIB_TABLE,
-                matches, instructions);
+                                                             getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE,
+                                                                                 NwConstants.TABLE_MISS_FLOW),
+                                                             NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow",
+                                                             0, 0, COOKIE_VM_FIB_TABLE,
+                                                             matches, instructions);
 
         if (addOrRemove == NwConstants.ADD_FLOW) {
             LOG.debug("Invoking MDSAL to install Table Miss Entries");
@@ -1538,88 +1855,97 @@ public class FibManager extends AbstractDataChangeListener<VrfEntry> implements
                 .append(FLOWID_PREFIX).toString();
     }
 
-    /*
-     * Install flow entry in protocol table to forward mpls
-     * coming through gre tunnel to LFIB table.
-     */
-    private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
-        final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
-        // Instruction to goto L3 InterfaceTable
-        List<InstructionInfo> instructions = new ArrayList<>();
-        instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
-        List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        matches.add(new MatchInfo(MatchFieldType.eth_type,
-                new long[] { NwConstants.ETHTYPE_MPLS_UC }));
-        FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_PROTOCOL_TABLE,
-                getTableMissFlowRef(dpnId, NwConstants.L3_PROTOCOL_TABLE,
-                        NwConstants.L3_LFIB_TABLE),
-                DEFAULT_FIB_FLOW_PRIORITY,
-                "Protocol Table For LFIB",
-                0, 0,
-                COOKIE_PROTOCOL_TABLE,
-                matches, instructions);
-
-        if (addOrRemove == NwConstants.ADD_FLOW) {
-            LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
-            mdsalManager.installFlow(flowEntityToLfib);
-        } else {
-            mdsalManager.removeFlow(flowEntityToLfib);
-        }
-    }
-
-    public List<String> printFibEntries() {
-        List<String> result = new ArrayList<String>();
-        result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
-        result.add("-------------------------------------------------------------------");
-        InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
-        Optional<FibEntries> fibEntries = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
-        if (fibEntries.isPresent()) {
-            List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
-            for (VrfTables vrfTable : vrfTables) {
-                for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
-                    for (String nextHop : vrfEntry.getNextHopAddressList()) {
-                        result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", vrfTable.getRouteDistinguisher(),
-                                vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
-                    }
+  /*
+   * Install flow entry in protocol table to forward mpls
+   * coming through gre tunnel to LFIB table.
+   */
+  private void makeProtocolTableFlow(BigInteger dpnId, int addOrRemove) {
+    final BigInteger COOKIE_PROTOCOL_TABLE = new BigInteger("1070000", 16);
+    // Instruction to goto L3 InterfaceTable
+    List<InstructionInfo> instructions = new ArrayList<>();
+    instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] {NwConstants.L3_LFIB_TABLE}));
+    List<MatchInfo> matches = new ArrayList<MatchInfo>();
+    matches.add(new MatchInfo(MatchFieldType.eth_type,
+                              new long[] { NwConstants.ETHTYPE_MPLS_UC }));
+    FlowEntity flowEntityToLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_PROTOCOL_TABLE,
+                                                          getTableMissFlowRef(dpnId, NwConstants.L3_PROTOCOL_TABLE,
+                                                                  NwConstants.L3_LFIB_TABLE),
+                                                          DEFAULT_FIB_FLOW_PRIORITY,
+                                                          "Protocol Table For LFIB",
+                                                          0, 0,
+                                                          COOKIE_PROTOCOL_TABLE,
+                                                          matches, instructions);
+
+    if (addOrRemove == NwConstants.ADD_FLOW) {
+      LOG.debug("Invoking MDSAL to install Protocol Entries for dpn {}", dpnId);
+      mdsalManager.installFlow(flowEntityToLfib);
+    } else {
+      mdsalManager.removeFlow(flowEntityToLfib);
+    }
+  }
+
+  public List<String> printFibEntries() {
+    List<String> result = new ArrayList<String>();
+    result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", "RD", "Prefix", "NextHop", "Label", "Origin"));
+    result.add("-------------------------------------------------------------------");
+    InstanceIdentifier<FibEntries> id = InstanceIdentifier.create(FibEntries.class);
+    Optional<FibEntries> fibEntries = FibUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+    if (fibEntries.isPresent()) {
+        List<VrfTables> vrfTables = fibEntries.get().getVrfTables();
+        for (VrfTables vrfTable : vrfTables) {
+            for (VrfEntry vrfEntry : vrfTable.getVrfEntry()) {
+                for (String nextHop : vrfEntry.getNextHopAddressList()) {
+                    result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", vrfTable.getRouteDistinguisher(),
+                            vrfEntry.getDestPrefix(), nextHop, vrfEntry.getLabel(), vrfEntry.getOrigin()));
+                }
+                if (vrfEntry.getNextHopAddressList().isEmpty()) {
+                    result.add(String.format("   %-7s  %-20s  %-20s  %-7s  %-7s", vrfTable.getRouteDistinguisher(),
+                            vrfEntry.getDestPrefix(), "local", vrfEntry.getLabel(), vrfEntry.getOrigin()));
                 }
             }
         }
-        return result;
     }
+    return result;
+  }
 
-    private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) {
-        List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
-        List<MatchInfo> matches = new ArrayList<MatchInfo>();
-        final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
-        // Instruction to goto L3 InterfaceTable
+  private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) {
+    List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
+    List<MatchInfo> matches = new ArrayList<MatchInfo>();
+    final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16);
+    // Instruction to goto L3 InterfaceTable
 
-        List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
-        actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
-                Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
-        instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
-        //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
+    List <ActionInfo> actionsInfos = new ArrayList <ActionInfo> ();
+    actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
+        Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
+    instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
+    //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE }));
 
-        FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
-                getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
-                NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
-        if (addOrRemove == NwConstants.ADD_FLOW) {
-            LOG.info("Invoking MDSAL to install L3 interface Table Miss Entries");
-            mdsalManager.installFlow(flowEntityL3Intf);
-        } else {
-            mdsalManager.removeFlow(flowEntityL3Intf);
-        }
+    FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE,
+            getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW),
+            NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions);
+    if (addOrRemove == NwConstants.ADD_FLOW) {
+      LOG.info("Invoking MDSAL to install L3 interface Table Miss Entries");
+      mdsalManager.installFlow(flowEntityL3Intf);
+    } else {
+      mdsalManager.removeFlow(flowEntityL3Intf);
     }
+  }
 
     private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
         InstanceIdentifier<VrfEntry> vrfEntryId =
-                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
-                        child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
+                    InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
+                            child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
         Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
         if (vrfEntry.isPresent())  {
             return (vrfEntry.get());
         }
         return null;
     }
-}
-
 
+    private InstanceIdentifier<VrfEntry> getVrfEntryId(String rd, String ipPrefix) {
+        InstanceIdentifier<VrfEntry> vrfEntryId =
+                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
+                        child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
+        return vrfEntryId;
+    }
+}
index 327da2f48d5792335d47ccd59e39f8927176e364..84d7f1c3c9a762a971e0d3a8ef52a4c5e5bfa4ea 100644 (file)
@@ -113,10 +113,9 @@ public class FibManagerProvider implements BindingAwareProvider, IFibManager, Au
     this.vpnmanager.addExtraRoute(prefix, nextHop, rd, null, label);
   }
 
-  //FIXME: Once changes are upstreamed from vpnmanager
   @Override
   public void deleteStaticRoute(String prefix, String nextHop, String rd) {
-  //  this.vpnmanager.delExtraRoute(prefix, nextHop, rd, null);
+    this.vpnmanager.delExtraRoute(prefix, nextHop, rd, null);
   }
 
   public void setRpcProviderRegistry(RpcProviderRegistry rpcProviderRegistry) {
@@ -159,17 +158,29 @@ public class FibManagerProvider implements BindingAwareProvider, IFibManager, Au
   }
 
   @Override
-  public void handleRemoteRoute(boolean action, BigInteger localDpnId, BigInteger remoteDpnId, long vpnId, String  rd, String destPrefix , String nextHopIp) {
-    fibManager.handleRemoteRoute(action, localDpnId, remoteDpnId, vpnId,rd, destPrefix, nextHopIp);
+  public void handleRemoteRoute(boolean action, BigInteger localDpnId,
+                                BigInteger remoteDpnId, long vpnId,
+                                String  rd, String destPrefix ,
+                                String localNextHopIP,
+                                String remoteNextHopIp) {
+    fibManager.handleRemoteRoute( action, localDpnId, remoteDpnId,
+                                  vpnId,rd, destPrefix,
+                                  localNextHopIP, remoteNextHopIp);
   }
   @Override
-  public void populateFibOnDpn(BigInteger localDpnId, BigInteger remoteDpnId, long vpnId, String rd, String nextHopIp) {
-    fibManager.populateFibOnDpn(localDpnId, remoteDpnId, vpnId, rd, nextHopIp);
+  public void populateFibOnDpn(BigInteger localDpnId, long vpnId,
+                               String rd, String localNextHopIp,
+                               String remoteNextHopIp) {
+    fibManager.populateFibOnDpn(localDpnId, vpnId, rd,
+                                localNextHopIp, remoteNextHopIp);
   }
 
   @Override
-  public void cleanUpDpnForVpn(BigInteger dpnId, long vpnId, String rd, String nextHopIp) {
-    fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd, nextHopIp);
+  public void cleanUpDpnForVpn(BigInteger dpnId, long vpnId,
+                               String rd, String localNextHopIp,
+                               String remoteNextHopIp) {
+    fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd,
+                                localNextHopIp, remoteNextHopIp);
   }
 
 }
\ No newline at end of file
index c290393373cfe40a95a1c4f2e13619ed25627b7b..5554b813e24946900d1b10ad4d6f6c29f12f2c54 100644 (file)
@@ -9,6 +9,7 @@
 package org.opendaylight.netvirt.fibmanager;
 
 import com.google.common.base.Optional;
+import com.google.common.util.concurrent.CheckedFuture;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 
@@ -16,14 +17,18 @@ import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
 import org.opendaylight.genius.mdsalutil.MDSALUtil;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.*;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnIdToVpnInstance;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceToVpnId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIdsKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
         .VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
@@ -80,8 +85,14 @@ public class FibUtil {
     static <T extends DataObject> void syncWrite(DataBroker broker, LogicalDatastoreType datastoreType,
                                                  InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
         WriteTransaction tx = broker.newWriteOnlyTransaction();
-        tx.merge(datastoreType, path, data, true);
-        tx.submit();
+        tx.put(datastoreType, path, data, true);
+        CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
+        try {
+            futures.get();
+        } catch (InterruptedException | ExecutionException e) {
+            LOG.error("Error writing to datastore (path, data) : ({}, {})", path, data);
+            throw new RuntimeException(e.getMessage());
+        }
     }
 
     static <T extends DataObject> void delete(DataBroker broker, LogicalDatastoreType datastoreType, InstanceIdentifier<T> path) {
@@ -406,4 +417,37 @@ public class FibUtil {
                 };
             };
 
+    public static String getVpnNameFromId(DataBroker broker, long vpnId) {
+
+        InstanceIdentifier<VpnIds> id
+                = getVpnIdToVpnInstanceIdentifier(vpnId);
+        Optional<VpnIds> vpnInstance
+                = read(broker, LogicalDatastoreType.CONFIGURATION, id);
+
+        String vpnName = null;
+        if (vpnInstance.isPresent()) {
+            vpnName = vpnInstance.get().getVpnInstanceName();
+        }
+        return vpnName;
+    }
+
+    static InstanceIdentifier<VpnIds>
+    getVpnIdToVpnInstanceIdentifier(long vpnId) {
+        return InstanceIdentifier.builder(VpnIdToVpnInstance.class)
+                .child(VpnIds.class, new VpnIdsKey(Long.valueOf(vpnId))).build();
+    }
+    
+    public static <T extends DataObject> void syncUpdate(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                         InstanceIdentifier<T> path, T data) {
+        WriteTransaction tx = broker.newWriteOnlyTransaction();
+        tx.put(datastoreType, path, data, true);
+        CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
+        try {
+            futures.get();
+        } catch (InterruptedException | ExecutionException e) {
+            LOG.error("Error writing to datastore (path, data) : ({}, {})", path, data);
+            throw new RuntimeException(e.getMessage());
+        }
+    }
+
 }
index 5364660ec03ef44d805261ef064993da25bcec88..fd157e605ec41860ee9dcd522d7538425e9f6559 100644 (file)
@@ -32,13 +32,13 @@ import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
 import org.opendaylight.netvirt.fibmanager.FibManager;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
@@ -83,7 +83,6 @@ public class FibManagerTest {
   BigInteger Dpn;
   private static final long vpnId = 101L;
   private static final long vpnIntfCnt = 2;
-  private static final Boolean isCleanupComplete = Boolean.FALSE;
 
   private void SetupMocks() {
     Dpn = BigInteger.valueOf(100000L);
@@ -131,9 +130,6 @@ public class FibManagerTest {
           @Override
           public Long getVpnInterfaceCount() { return vpnIntfCnt; }
 
-          @Override
-          public Boolean isCleanupComplete(){return isCleanupComplete;}
-
           @Override
           public List<VpnToDpnList> getVpnToDpnList() {
             List <VpnToDpnList> vpnToDpnLists =  new ArrayList<>();
index 94fe57899978a26397243a57c5ba2313c197eb6b..62f210d5667da9f943e7596abe0c0e06536edf86 100644 (file)
@@ -60,8 +60,13 @@ and is available at http://www.eclipse.org/legal/epl-v10.html
       <version>${vpnservices.version}</version>
     </dependency>
     <dependency>
-        <groupId>org.opendaylight.controller</groupId>
-        <artifactId>config-api</artifactId>
+      <groupId>org.opendaylight.netvirt</groupId>
+      <artifactId>bgpmanager-api</artifactId>
+      <version>${vpnservices.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>config-api</artifactId>
     </dependency>
     <dependency>
       <groupId>org.opendaylight.genius</groupId>
index 524fa93904c4952a439b7010f86d41f4988db8e2..d5dafb0e73a252b92fc35bad1e6d01b104af998c 100644 (file)
@@ -13,7 +13,7 @@ import org.opendaylight.netvirt.fibmanager.api.IFibManager;
 public interface IVpnManager {
     void setFibService(IFibManager fibManager);
     void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label);
-    void delExtraRoute(String destination, String rd, String routerID);
+    void delExtraRoute(String destination, String nextHop, String rd, String routerID);
 
     /**
      * Returns true if the specified VPN exists
index 0639588d2e41babf312321ac1cc6e5a01a329d8b..ffb32a781dfe5f3c072639add2604b2f5b00a923 100644 (file)
@@ -5,7 +5,7 @@ module inter-vpn-link {
 
   import ietf-inet-types { prefix inet; revision-date "2013-07-15"; }
 
-  import ietf-yang-types { prefix yang; }
+  import ietf-yang-types { prefix yang; revision-date "2013-07-15"; }
 
   import config { prefix config; revision-date 2013-04-05; }
 
index bea1c24daedfd059cd10431eed272c89cb464742..ffd8e00a2fac0ad3e0ce16f55be6ed946181611f 100644 (file)
@@ -784,6 +784,12 @@ module l3vpn {
           length "1..40";
         }
       }
+      leaf dpn-id {
+        type uint64;
+      }
+      leaf scheduled-for-remove {
+        type boolean;
+      }
     }
   }
 
index 70afecb88eeaf71d885d46c7e4d2d635878c7cdd..31e64c87e1495c1d48dbecc35e4f9369995a3be5 100644 (file)
@@ -4,7 +4,7 @@ module odl-l3vpn {
 
     import yang-ext {prefix ext; revision-date "2013-07-09";}
     import l3vpn { prefix l3vpn; revision-date "2014-08-15"; }
-    import ietf-yang-types { prefix "yang"; }
+    import ietf-yang-types { prefix "yang"; revision-date "2013-07-15"; }
     import odl-interface { prefix odlif; revision-date "2016-04-06"; }
 
     revision "2013-09-11" {
@@ -70,7 +70,7 @@ module odl-l3vpn {
            list extraroute {
               key prefix;
               leaf prefix {type string;}
-              leaf nexthop-ip {
+              leaf-list nexthop-ip-list {
                   type string;
               }
            }
@@ -97,14 +97,48 @@ module odl-l3vpn {
        }
     }
 
+    container vpn-id-to-vpn-instance {
+           list vpn-ids {
+              key vpn-id;
+              leaf vpn-id {
+                 type uint32;
+              }
+              leaf vpn-instance-name {
+                 type string;
+              }
+              leaf vrf-id {
+                  description
+                     "The vrf-id command configures a route distinguisher (RD)
+                      for the IPv4 or IPv6 address family of a VPN instance or
+                      vpn instance name for internal vpn case.";
+                  type string;
+              }
+              leaf external-vpn {
+                  type boolean;
+                  description "The VPN is external?";
+              }
+           }
+    }
+
+    container vpn-to-pseudo-port-tag-data {
+        description "Makes a correspondence between VRFs and their corresponding VpnPseudoPort";
+        list vpn-to-pseudo-port-tag {
+            key vrf-id;
+            leaf vrf-id {
+                type string;
+            }
+            leaf lport-tag {
+                description "Vpn Pseudo Logical Port Tag";
+                type uint32;
+            }
+        }
+    }
+
     container vpn-instance-op-data {
         config false;
         list vpn-instance-op-data-entry {
            key vrf-id;
            leaf vpn-id { type uint32;}
-           leaf vpn-instance-name {
-              type string;
-           }
            leaf vrf-id {
               description
                  "The vrf-id command configures a route distinguisher (RD)
@@ -113,6 +147,11 @@ module odl-l3vpn {
               type string;
            }
 
+           leaf vpn-instance-name {
+               description "Typical the VPN Uuid";
+               type string;
+           }
+
            leaf vpn-interface-count { type uint32; }
            uses vpn-route-list;
            list vpn-to-dpn-list {
@@ -127,16 +166,14 @@ module odl-l3vpn {
                   }
                }
                list ip-addresses {
-                  key ip-address;
-                  leaf ip-address {
-                      type string;
-                  }
+                   key ip-address;
+                   leaf ip-address { type string; }
                }
            }
-           leaf cleanup_complete { type boolean;}
         }
     }
 
+
     typedef task-state {
          type enumeration {
            enum na {
@@ -159,7 +196,7 @@ module odl-l3vpn {
           "This value the status of any task.
            The possible values are NA, PENDING or DONE.
            ";
-    }
+           }
 
 
     container subnet-op-data {
@@ -226,24 +263,39 @@ module odl-l3vpn {
         }
     }
 
+
     grouping dpn-in-vpn-event {
-            leaf dpn-id { type uint64; }
-            leaf vpn-name { type string; }
-            leaf rd { type string; }
+        leaf dpn-id { type uint64; }
+        leaf vpn-name { type string; }
+        leaf rd { type string; }
+    }
+
+    notification add-dpn-event {
+        container add-event-data {
+           uses dpn-in-vpn-event;
         }
+    }
 
-        notification add-dpn-event {
-            container add-event-data {
-               uses dpn-in-vpn-event;
-            }
+    notification remove-dpn-event {
+        container remove-event-data {
+           uses dpn-in-vpn-event;
         }
+    }
 
-        notification remove-dpn-event {
-            container remove-event-data {
-               uses dpn-in-vpn-event;
+    /*
+     * Configured Transport Type for l3vpn service.
+     */
+    container conf-transport-type-l3vpn {
+        leaf transport-type {
+            mandatory "true";
+            type identityref {
+                base odlif:tunnel-type-base;
             }
+            description
+                "L3VPN service will use this config to setup
+                the transport type for tunnels between DPNs.";
         }
-
+    }
 
     /* container to maintain mapping between neutron router and DPN(s) on which vpn-interfaces for router are present */
     container neutron-router-dpns {
@@ -262,6 +314,7 @@ module odl-l3vpn {
         }
     }
 
+
     container router-interfaces {
         list router-interface {
             key interface-name;
@@ -269,19 +322,4 @@ module odl-l3vpn {
             leaf router-name { type string; }
         }
     }
-
-    /*
-     * Configured Transport Type for l3vpn service.
-     */
-    container conf-transport-type-l3vpn {
-        leaf transport-type {
-            mandatory "true";
-            type identityref {
-                base odlif:tunnel-type-base;
-            }
-            description
-                "L3VPN service will use this config to setup
-                the transport type for tunnels between DPNs.";
-        }
-    }
 }
index 175b3e6441e4b949e51afc377209186fc5c53e3a..223ba6afbba3262a259b6c7268578a6548e3693e 100644 (file)
@@ -7,6 +7,57 @@ module vpn-rpc {
     }
 
     /* RPCs */
+    rpc add-static-route {
+        description "Creates a static route in a VPN, creating a new label if needed";
+        input {
+            leaf vpnInstanceName {
+                mandatory true;
+                type string;
+            }
+            leaf destination {
+                mandatory true;
+                type string;
+            }
+            leaf nexthop {
+                description "Ip-address of the nexthop. The RPC will check if this nexthop is the endpoint
+                             of a VPN in an InterVpnLink";
+                mandatory true;
+                type string;
+            }
+            leaf label {
+                description "Label for the static route. It can be left empty, in that case a new
+                            label will be autogenerated";
+                type uint32;
+            }
+        }
+        output {
+            leaf label {
+                description "The auto-generated label, if applies";
+                type uint32;
+            }
+        }
+    }
+
+    rpc remove-static-route {
+        description "Removes a static route in a VPN, identified by this vpn name, destination and nexthop.
+                     Nextop is optional. If specified, the RPC will just remove one of the nexthops in the
+                     route (ECMP feature). If not specified, the whole route will be removed";
+        input {
+            leaf vpnInstanceName {
+                mandatory true;
+                type string;
+            }
+            leaf destination {
+                mandatory true;
+                type string;
+            }
+            leaf nexthop {
+                description "Ip-address of the nexthop. If specified, just one nexthop will be removed. Removing
+                             the last nexthop in a route, will provoke the removal of the whole route";
+                type string;
+            }
+        }
+    }
 
     rpc add-static-route {
         description "Creates a static route in a VPN, creating a new label if needed";
index eb16fffe883d82be83b4752347e5b49f6a9b684c..fd4ba453975b582e730dff00991ea1b6607e8c5a 100644 (file)
@@ -65,16 +65,21 @@ and is available at http://www.eclipse.org/legal/epl-v10.html
       <groupId>${project.groupId}</groupId>
       <artifactId>neutronvpn-api</artifactId>
       <version>${vpnservices.version}</version>
+      </dependency>
+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>sal-binding-broker-impl</artifactId>
     </dependency>
+    <!--dependency>
+      <groupId>org.opendaylight.vpnservice</groupId>
+      <artifactId>lockmanager-api</artifactId>
+      <version>${vpnservices.version}</version>
+    </dependency-->
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>elanmanager-api</artifactId>
       <version>${vpnservices.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.opendaylight.controller</groupId>
-      <artifactId>sal-binding-broker-impl</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.opendaylight.netvirt</groupId>
       <artifactId>natservice-api</artifactId>
index 9fbffd0fa22adf984da9b069ed875da84e285326..7d8255da842e8d3a791e70ad1e83e3ac7463a5c3 100644 (file)
@@ -41,6 +41,10 @@ and is available at http://www.eclipse.org/legal/epl-v10.html
             <type xmlns:bindingimpl="urn:opendaylight:params:xml:ns:yang:controller:md:sal:binding:impl">bindingimpl:binding-new-notification-service</type>
             <name>binding-notification-adapter</name>
           </notification-service>
+          <notification-publish-service>
+            <type xmlns:bindingimpl="urn:opendaylight:params:xml:ns:yang:controller:md:sal:binding:impl">bindingimpl:binding-new-notification-publish-service</type>
+            <name>binding-notification-publish-adapter</name>
+          </notification-publish-service>
         </module>
       </modules>
       <services xmlns="urn:opendaylight:params:xml:ns:yang:controller:config">
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpConstants.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpConstants.java
new file mode 100644 (file)
index 0000000..bac53e8
--- /dev/null
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager;
+
+public class ArpConstants {
+
+        public static final int THREAD_POOL_SIZE = 5;
+        public static final int NO_DELAY = 0;
+        public static long arpCacheTimeout;
+        public static final int RETRY_COUNT = 5;
+        public static final short ARP_REQUEST_OP = (short) 1;
+        public static final short ETH_TYPE_ARP = 0x0806;
+        public static final String PREFIX = "/32";
+        public static final String NODE_CONNECTOR_NOT_FOUND_ERROR = "Node connector id not found for interface %s";
+        public static final String FAILED_TO_GET_SRC_IP_FOR_INTERFACE = "Failed to get src ip for %s";
+        public static final String FAILED_TO_GET_SRC_MAC_FOR_INTERFACE = "Failed to get src mac for interface %s iid %s ";
+        public static final int PERIOD = 10000;
+
+}
index 349ee8633f035858153556dfa48663cc8b0772ca..5e1799f94ef82e0c91312634948e73dca7b98dc6 100644 (file)
@@ -7,16 +7,32 @@
  */
 package org.opendaylight.netvirt.vpnmanager;
 
+import com.google.common.base.Optional;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.genius.mdsalutil.MetaDataUtil;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.ArpRequestReceived;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.ArpResponseReceived;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.MacChanged;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilListener;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.PortKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.subnets.rev150712.subnets.attributes.Subnets;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.subnets.rev150712.subnets.attributes.subnets.Subnet;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.subnets.rev150712.subnets.attributes.subnets.SubnetKey;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.math.BigInteger;
+
 public class ArpNotificationHandler implements OdlArputilListener {
 
     VpnInterfaceManager vpnIfManager;
@@ -34,20 +50,157 @@ public class ArpNotificationHandler implements OdlArputilListener {
     }
 
     public void onArpRequestReceived(ArpRequestReceived notification){
+        LOG.trace("ArpNotification Request Received from interface {} and IP {} having MAC {} target destination {}",
+                notification.getInterface(), notification.getSrcIpaddress().getIpv4Address().getValue(),
+                notification.getSrcMac().getValue(),notification.getDstIpaddress().getIpv4Address().getValue());
         String srcInterface = notification.getInterface();
         IpAddress srcIP = notification.getSrcIpaddress();
         PhysAddress srcMac = notification.getSrcMac();
         IpAddress targetIP = notification.getDstIpaddress();
+        BigInteger metadata = notification.getMetadata();
+        if (metadata != null && metadata != BigInteger.ZERO) {
+            long vpnId = MetaDataUtil.getVpnIdFromMetadata(metadata);
+            // Respond to ARP request only if vpnservice is configured on the interface
+            if (VpnUtil.isVpnInterfaceConfigured(broker, srcInterface)) {
+                LOG.info("Received ARP Request for interface {} ", srcInterface);
+                InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds>
+                        vpnIdsInstanceIdentifier = VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId);
+                Optional<VpnIds> vpnIdsOptional
+                        = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnIdsInstanceIdentifier);
+                if (!vpnIdsOptional.isPresent()) {
+                    // Donot respond to ARP requests on unknown VPNs
+                    LOG.trace("ARP NO_RESOLVE: VPN {} not configured. Ignoring responding to ARP requests on this VPN", vpnId);
+                    return;
+                }
+                String vpnName = vpnIdsOptional.get().getVpnInstanceName();
+                String ipToQuery = notification.getSrcIpaddress().getIpv4Address().getValue();
+                LOG.trace("ArpRequest being processed for Source IP {}", ipToQuery);
+                VpnIds vpnIds = vpnIdsOptional.get();
+                VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnIds.getVpnInstanceName(), ipToQuery);
+                if (vpnPortipToPort != null) {
+                    String oldPortName = vpnPortipToPort.getPortName();
+                    String oldMac = vpnPortipToPort.getMacAddress();
+                    if (!oldMac.equalsIgnoreCase(srcMac.getValue())) {
+                        //MAC has changed for requested IP
+                        LOG.trace("ARP request Source IP/MAC data etmodified for IP {} with MAC {} and Port {}", ipToQuery,
+                                srcMac, srcInterface);
+                        if (!vpnPortipToPort.isConfig()) {
+                            VpnUtil.updateVpnPortFixedIpToPort(broker, vpnName, ipToQuery, srcInterface,
+                                    srcMac.getValue(), false, false, true);
+                            vpnIfManager.removeMIPAdjacency(vpnName, oldPortName, srcIP);
+                            try {
+                                Thread.sleep(2000);
+                            } catch (Exception e) {
+                            }
+                            vpnIfManager.addMIPAdjacency(vpnName, srcInterface, srcIP);
+                        } else {
+                            //MAC mismatch for a Neutron learned IP
+                            LOG.warn("MAC Address mismatach for Interface {} having a Mac  {},  IP {} and Arp learnt Mac {}",
+                                    oldPortName, oldMac, ipToQuery, srcMac.getValue());
+                            return;
+                        }
+                    }
+                } else {
+                    VpnUtil.createVpnPortFixedIpToPort(broker, vpnName, ipToQuery, srcInterface, srcMac.getValue(), false, false, true);
+                    vpnIfManager.addMIPAdjacency(vpnName, srcInterface, srcIP);
+                }
+                String targetIpToQuery = notification.getDstIpaddress().getIpv4Address().getValue();
+                VpnPortipToPort vpnTargetIpToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker,
+                        vpnIds.getVpnInstanceName(), targetIpToQuery);
+                //Process and respond from Controller only for GatewayIp ARP request
+                if (vpnTargetIpToPort != null) {
+                    if (vpnTargetIpToPort.isSubnetIp()) {
+                        String macAddress = vpnTargetIpToPort.getMacAddress();
+                        PhysAddress targetMac = new PhysAddress(macAddress);
+                        vpnIfManager.processArpRequest(srcIP, srcMac, targetIP, targetMac, srcInterface);
+                    }
+                } else {
+                    //Respond for gateway Ips ARP requests if L3vpn configured without a router
+                    if( vpnIds.isExternalVpn()) {
+                        Port prt;
+                        String gw = null;
+                        Uuid portUuid = new Uuid(srcInterface);
+                        InstanceIdentifier<Port> inst = InstanceIdentifier.create(Neutron.class)
+                                .child(Ports.class)
+                                .child(Port.class, new PortKey(portUuid));
+                        Optional<Port> port = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
+                        if (port.isPresent()) {
+                            prt = port.get();
+                            Uuid subnetUUID = prt.getFixedIps().get(0).getSubnetId();
+                            LOG.trace("Subnet UUID for this VPN Interface is {}", subnetUUID);
+                            SubnetKey subnetkey = new SubnetKey(subnetUUID);
+                            InstanceIdentifier<Subnet> subnetidentifier = InstanceIdentifier.create(Neutron.class)
+                                    .child(Subnets.class)
+                                    .child(Subnet.class, subnetkey);
+                            Optional<Subnet> subnet = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subnetidentifier);
+                            if (subnet.isPresent()) {
+                                gw = subnet.get().getGatewayIp().getIpv4Address().getValue();
+                                if (targetIpToQuery.equalsIgnoreCase(gw)) {
+                                    LOG.trace("Target Destination matches the Gateway IP {} so respond for ARP", gw);
+                                    vpnIfManager.processArpRequest(srcIP, srcMac, targetIP, null, srcInterface);
+                                }
+                            }
+                        }
+                    } else {
+                        LOG.trace("ARP request is not on an External VPN, so ignoring the request.");
+                    }
+
+                }
 
-        // Respond to ARP request only if vpnservice is configured on the interface
-        if(VpnUtil.isVpnInterfaceConfigured(broker, srcInterface)) {
-            LOG.info("Received ARP Request for interface {} ", srcInterface);
-            vpnIfManager.processArpRequest(srcIP, srcMac, targetIP, null, srcInterface);
+            }
         }
     }
-     
     public void onArpResponseReceived(ArpResponseReceived notification){
-
+        LOG.trace("ArpNotification Response Received from interface {} and IP {} having MAC {}",notification.getInterface(),
+                notification.getIpaddress().getIpv4Address().getValue(), notification.getMacaddress().getValue());
+        String srcInterface = notification.getInterface();
+        IpAddress srcIP = notification.getIpaddress();
+        PhysAddress srcMac = notification.getMacaddress();
+        BigInteger metadata = notification.getMetadata();
+        if (metadata != null && metadata != BigInteger.ZERO) {
+            long vpnId = MetaDataUtil.getVpnIdFromMetadata(metadata);
+            InstanceIdentifier<VpnIds>
+                    vpnIdsInstanceIdentifier = VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId);
+            Optional<VpnIds> vpnIdsOptional
+                    = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnIdsInstanceIdentifier);
+            if (!vpnIdsOptional.isPresent()) {
+                // Donot respond to ARP requests on unknown VPNs
+                LOG.trace("ARP NO_RESOLVE: VPN {} not configured. Ignoring responding to ARP requests on this VPN", vpnId);
+                return;
+            }
+            if (VpnUtil.isVpnInterfaceConfigured(broker, srcInterface)) {
+                String vpnName = vpnIdsOptional.get().getVpnInstanceName();
+                String ipToQuery = notification.getIpaddress().getIpv4Address().getValue();
+                VpnIds vpnIds = vpnIdsOptional.get();
+                VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnIds.getVpnInstanceName(), ipToQuery);
+                if (vpnPortipToPort != null) {
+                    String oldMac = vpnPortipToPort.getMacAddress();
+                    String oldPortName = vpnPortipToPort.getPortName();
+                    if (!oldMac.equalsIgnoreCase(srcMac.getValue())) {
+                        //MAC has changed for requested IP
+                        LOG.trace("ARP response Source IP/MAC data modified for IP {} with MAC {} and Port {}", ipToQuery,
+                                srcMac, srcInterface);
+                        if (!vpnPortipToPort.isConfig()) {
+                            VpnUtil.updateVpnPortFixedIpToPort(broker, vpnName, ipToQuery, srcInterface,
+                                    srcMac.getValue(), false, false, true);
+                            vpnIfManager.removeMIPAdjacency(vpnName, oldPortName, srcIP);
+                            try {
+                                Thread.sleep(2000);
+                            } catch (Exception e) {
+                            }
+                            vpnIfManager.addMIPAdjacency(vpnName, srcInterface, srcIP);
+                        } else {
+                            //MAC mismatch for a Neutron learned IP set learnt back to false
+                            LOG.warn("MAC Address mismatch for Interface {} having a Mac  {} , IP {} and Arp learnt Mac {}",
+                                    srcInterface, oldMac, ipToQuery, srcMac.getValue());
+                        }
+                    }
+                } else {
+                    VpnUtil.createVpnPortFixedIpToPort(broker, vpnName, ipToQuery, srcInterface, srcMac.getValue(), false, false, true);
+                    vpnIfManager.addMIPAdjacency(vpnName, srcInterface, srcIP);
+                }
+            }
+        }
     }
 
 }
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpScheduler.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/ArpScheduler.java
new file mode 100644 (file)
index 0000000..4d2c88b
--- /dev/null
@@ -0,0 +1,270 @@
+/*
+ * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.binding.api.DataTreeIdentifier;
+import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
+import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstanceKey;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronVpnPortipPortData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPortKey;
+import org.opendaylight.yangtools.concepts.ListenerRegistration;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.*;
+
+public class ArpScheduler extends AsyncDataTreeChangeListenerBase<VpnPortipToPort,ArpScheduler> {
+
+    private static final Logger logger = LoggerFactory.getLogger(ArpScheduler.class);
+    private ScheduledExecutorService executorService;
+    private OdlInterfaceRpcService interfaceRpc;
+    private DataBroker dataBroker;
+    private ScheduledFuture<?> scheduledResult;
+    private static final Logger LOG = LoggerFactory.getLogger(ArpScheduler.class);
+    private DelayQueue<MacEntry> macEntryQueue = new DelayQueue<MacEntry>();
+    private ListenerRegistration<ArpScheduler> listenerRegistration;
+
+    public ArpScheduler(OdlInterfaceRpcService interfaceRpc, DataBroker dataBroker) {
+        super(VpnPortipToPort.class, ArpScheduler.class);
+        this.dataBroker = dataBroker;
+        this.interfaceRpc = interfaceRpc;
+        Long timeout = Long.getLong("arp.cache.timeout");
+        if (timeout == null) {
+            timeout = 60000L;
+        }
+        ArpConstants.arpCacheTimeout = timeout;
+        registerListener(this.dataBroker);
+        executorService = Executors.newScheduledThreadPool(ArpConstants.THREAD_POOL_SIZE, getThreadFactory("Arp Cache Timer Tasks"));
+        scheduleExpiredEntryDrainerTask();
+    }
+
+    public void addOrUpdateMacEntryToQueue(String vpnName, MacAddress macAddress, InetAddress InetAddress, String interfaceName) {
+        MacEntry newMacEntry = new MacEntry(ArpConstants.arpCacheTimeout,vpnName,macAddress, InetAddress,interfaceName );
+        if (!macEntryQueue.contains(newMacEntry)) {
+            LOG.info("Adding ARP cache");
+            macEntryQueue.offer(newMacEntry);
+        }
+        else{
+            LOG.info("Updating ARP cache");
+            macEntryQueue.remove(newMacEntry);
+            macEntryQueue.offer(newMacEntry);        }
+    }
+
+
+    private void scheduleExpiredEntryDrainerTask() {
+        LOG.info("Scheduling expired entry drainer task");
+        ExpiredEntryDrainerTask expiredEntryDrainerTask = new ExpiredEntryDrainerTask();
+        scheduledResult = executorService.scheduleAtFixedRate(expiredEntryDrainerTask, ArpConstants.NO_DELAY, ArpConstants.PERIOD, TimeUnit.MILLISECONDS);
+    }
+
+
+    private ThreadFactory getThreadFactory(String threadNameFormat) {
+        ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+        builder.setNameFormat(threadNameFormat);
+        builder.setUncaughtExceptionHandler( new UncaughtExceptionHandler() {
+            @Override
+            public void uncaughtException(Thread t, Throwable e) {
+                LOG.error("Received Uncaught Exception event in Thread: {}", t.getName(), e);
+          }
+       });
+      return builder.build();
+     }
+
+
+     private class ExpiredEntryDrainerTask implements Runnable {
+        @Override
+        public void run() {
+            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
+            Collection<MacEntry> expiredMacEntries = new ArrayList<>();
+            macEntryQueue.drainTo(expiredMacEntries);
+            for (MacEntry macEntry: expiredMacEntries) {
+                LOG.info("Removing the ARP cache for"+macEntry);
+                InstanceIdentifier<VpnPortipToPort> id = getVpnPortipToPortInstanceOpDataIdentifier(macEntry.getIpAddress().getHostAddress(),macEntry.getVpnName());
+                Optional<VpnPortipToPort> vpnPortipToPort = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+                if (vpnPortipToPort.isPresent()) {
+                    VpnPortipToPort vpnPortipToPortold = vpnPortipToPort.get();
+                    String fixedip = vpnPortipToPortold.getPortFixedip();
+                    String vpnName =  vpnPortipToPortold.getVpnName();
+                    String interfaceName =  vpnPortipToPortold.getPortName();
+                    String rd = getRouteDistinguisher(vpnName);
+                    deleteVrfEntries(rd,fixedip,tx);
+                    deleteAdjacencies(fixedip,vpnName,interfaceName,tx);
+                    tx.delete(LogicalDatastoreType.CONFIGURATION, id);
+                    waitForTransactionToComplete(tx);
+                 }
+
+              }
+        }
+     }
+        private void deleteVrfEntries(String rd, String fixedip, WriteTransaction tx) {
+            InstanceIdentifier<VrfEntry> vrfid= InstanceIdentifier.builder(FibEntries.class).
+                    child(VrfTables.class, new VrfTablesKey(rd)).
+                    child(VrfEntry.class,new VrfEntryKey(iptoprefix(fixedip))).
+                    build();
+
+            tx.delete(LogicalDatastoreType.CONFIGURATION, vrfid);
+            LOG.info("deleting the vrf entries");
+
+
+        }
+
+
+    public void deleteAdjacencies(String fixedip, String vpnName, String interfaceName, WriteTransaction tx) {
+          InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
+          InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
+          Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
+          if (adjacencies.isPresent()) {
+              List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
+              InstanceIdentifier <Adjacency> adid = vpnIfId.augmentation(Adjacencies.class).child(Adjacency.class, new AdjacencyKey(iptoprefix(fixedip)));
+              Optional<Adjacency> newAdj = VpnUtil.read(dataBroker,  LogicalDatastoreType.CONFIGURATION, adid);
+              if(adjacencyList.contains(newAdj.get()))
+              adjacencyList.remove(newAdj.get());
+              Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
+              VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(interfaceName)).
+                    setName(interfaceName).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
+              tx.put(LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf,true);
+              LOG.info("deleting the adjacencies ");
+           }
+    }
+    public static void waitForTransactionToComplete(WriteTransaction tx) {
+        CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
+        try {
+            futures.get();
+        } catch (InterruptedException | ExecutionException e) {
+            logger.error("Error writing to datastore {}", e);
+        }
+    }
+    private String iptoprefix(String ip){
+        return new StringBuilder(ip).append(ArpConstants.PREFIX).toString();
+
+     }
+
+    private static final FutureCallback<Void> DEFAULT_CALLBACK =
+        new FutureCallback<Void>() {
+        @Override
+        public synchronized void onSuccess(Void result) {
+             LOG.debug("Success in Datastore operation");
+        }
+
+        @Override
+        public void onFailure(Throwable error) {
+            LOG.error("Error in Datastore operation", error);
+        };
+    };
+    private String getRouteDistinguisher(String vpnName) {
+        InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
+                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
+        Optional<VpnInstance> vpnInstance = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
+        String rd = "";
+        if(vpnInstance.isPresent()) {
+            VpnInstance instance = vpnInstance.get();
+            VpnAfConfig config = instance.getIpv4Family();
+            rd = config.getRouteDistinguisher();
+         }
+         return rd;
+    }
+
+     public static InstanceIdentifier<VpnPortipToPort> getVpnPortipToPortInstanceOpDataIdentifier(String ip, String vpnName) {
+           return InstanceIdentifier.builder(NeutronVpnPortipPortData.class)
+               .child(VpnPortipToPort.class, new VpnPortipToPortKey(ip,vpnName)).build();
+    }
+
+     private void registerListener(final DataBroker db) {
+       try {
+            final DataTreeIdentifier<VpnPortipToPort> treeId = new DataTreeIdentifier<>(LogicalDatastoreType.CONFIGURATION, getWildCardPath());
+            listenerRegistration = dataBroker.registerDataTreeChangeListener(treeId, this);
+            } catch (final Exception e) {
+             LOG.error("VPN Service DataChange listener registration fail !", e);
+             throw new IllegalStateException("VPN Service registration Listener failed.", e);
+        }
+       }
+
+     @Override
+       protected ArpScheduler getDataTreeChangeListener() {
+        return this;
+    }
+
+    @Override
+    protected void update(InstanceIdentifier<VpnPortipToPort> id, VpnPortipToPort value,
+            VpnPortipToPort dataObjectModificationAfter) {
+        try {
+            InetAddress srcInetAddr = InetAddress.getByName(value.getPortFixedip());
+            MacAddress srcMacAddress = MacAddress.getDefaultInstance(value.getMacAddress());
+            String vpnName =  value.getVpnName();
+            String interfaceName =  value.getPortName();
+            addOrUpdateMacEntryToQueue(vpnName,srcMacAddress, srcInetAddr, interfaceName);
+           } catch (Exception e) {
+            LOG.error("Error in deserializing packet {} with exception {}", value, e);
+           e.printStackTrace();
+        }
+
+    }
+
+
+    @Override
+    protected void add(InstanceIdentifier<VpnPortipToPort> identifier, VpnPortipToPort value) {
+
+        try {
+            InetAddress srcInetAddr = InetAddress.getByName(value.getPortFixedip());
+            MacAddress srcMacAddress = MacAddress.getDefaultInstance(value.getMacAddress());
+            String vpnName =  value.getVpnName();
+            String interfaceName =  value.getPortName();
+            Boolean islearnt = value.isLearnt();
+            if(islearnt)
+            {
+               addOrUpdateMacEntryToQueue(vpnName,srcMacAddress, srcInetAddr, interfaceName);
+            }
+        }
+        catch (Exception e) {
+             LOG.error("Error in deserializing packet {} with exception {}", value, e);
+        }
+
+    }
+
+    @Override
+    protected InstanceIdentifier<VpnPortipToPort> getWildCardPath() {
+       return InstanceIdentifier.create(NeutronVpnPortipPortData.class).child(VpnPortipToPort.class);
+    }
+
+    @Override
+    protected void remove(InstanceIdentifier<VpnPortipToPort> key, VpnPortipToPort dataObjectModification) {
+       // TODO Auto-generated method stub
+
+     }
+}
index 0b8fcc6c33ddc69a37d5f896a16748244301d524..9adccd84d123e536aec36fef37404a11b3bd5896 100644 (file)
@@ -14,6 +14,7 @@ import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfacesState;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
@@ -42,9 +43,9 @@ public class InterfaceStateChangeListener extends AbstractDataChangeListener<Int
         registerListener(db);
     }
 
-    public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
+    public void setIfaceMgrRpcService(OdlInterfaceRpcService interfaceManager) {
       this.interfaceManager = interfaceManager;
-  }
+    }
 
     @Override
     public void close() throws Exception {
@@ -78,20 +79,30 @@ public class InterfaceStateChangeListener extends AbstractDataChangeListener<Int
         LOG.info("Received port UP event for interface {} ", interfaceName);
         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface
             configInterface = InterfaceUtils.getInterface(broker, interfaceName);
-        BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(intrf);
-        if (configInterface != null && configInterface.getType().equals(Tunnel.class)) {
-          if(intrf.getOperStatus().equals(Interface.OperStatus.Up)) {
-            //advertise all prefixes in all vpns for this dpn to bgp
-            // FIXME: Blocked until tunnel event[vxlan/gre] support is available
-            // vpnInterfaceManager.updatePrefixesForDPN(dpnId, VpnInterfaceManager.UpdateRouteAction.ADVERTISE_ROUTE);
-          }
-        } else {
-          vpnInterfaceManager.processVpnInterfaceUp(dpnId, interfaceName, intrf.getIfIndex());
-          vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceUp(intrf);
-          handleRouterInterfacesUpEvent(interfaceName);
+        BigInteger dpnId = BigInteger.ZERO;
+        try{
+            dpnId = InterfaceUtils.getDpIdFromInterface(intrf);
+        }catch(Exception e){
+            LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", intrf.getName(), e);
+            return;
+        }
+        if (configInterface != null) {
+            if (!configInterface.getType().equals(Tunnel.class)) {
+                // We service only VM interfaces and Router interfaces here.
+                // We donot service Tunnel Interfaces here.
+                // Tunnel events are directly serviced
+                // by TunnelInterfacesStateListener present as part of VpnInterfaceManager
+                final VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
+                if (vpnInterface != null) {
+                    vpnInterfaceManager.processVpnInterfaceUp(dpnId, interfaceName, intrf.getIfIndex(), false);
+                    vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceUp(intrf);
+                    handleRouterInterfacesUpEvent(interfaceName);
+                }
+
+            }
         }
       } catch (Exception e) {
-        LOG.error("Exception caught in Interface Operational State Up event", e);
+        LOG.error("Exception observed in handling addition for VPN Interface {}. ", intrf.getName(), e);
       }
     }
 
@@ -106,20 +117,30 @@ public class InterfaceStateChangeListener extends AbstractDataChangeListener<Int
       try {
         String interfaceName = intrf.getName();
         LOG.info("Received port DOWN event for interface {} ", interfaceName);
-        BigInteger dpId = InterfaceUtils.getDpIdFromInterface(intrf);
+        InstanceIdentifier<VpnInterface> id = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
+        Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
+        if (!existingVpnInterface.isPresent()) {
+            LOG.error("VPN Interface operational instance not available for interface {}, ignoring interface", interfaceName);
+            return;
+        }
         if (intrf != null && intrf.getType() != null && intrf.getType().equals(Tunnel.class)) {
-          //withdraw all prefixes in all vpns for this dpn from bgp
-          // FIXME: Blocked until tunnel event[vxlan/gre] support is available
-          // vpnInterfaceManager.updatePrefixesForDPN(dpId, VpnInterfaceManager.UpdateRouteAction.WITHDRAW_ROUTE);
+            //withdraw all prefixes in all vpns for this dpn from bgp
+            // FIXME: Blocked until tunnel event[vxlan/gre] support is available
+            // vpnInterfaceManager.updatePrefixesForDPN(dpId, VpnInterfaceManager.UpdateRouteAction.WITHDRAW_ROUTE);
         } else {
-          if (VpnUtil.isVpnInterfaceConfigured(broker, interfaceName)) {
-            vpnInterfaceManager.processVpnInterfaceDown(dpId, interfaceName, intrf.getIfIndex(), true);
+            BigInteger dpId = BigInteger.ZERO;
+            try{
+                dpId = InterfaceUtils.getDpIdFromInterface(intrf);
+            }catch(Exception e){
+                LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", intrf.getName(), e);
+                dpId = existingVpnInterface.get().getDpnId();
+            }
+            vpnInterfaceManager.processVpnInterfaceDown(dpId, interfaceName, intrf.getIfIndex(), false, false);
             vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceDown(intrf);
             handleRouterInterfacesDownEvent(interfaceName,dpId);
-          }
         }
       } catch (Exception e) {
-        LOG.error("Exception caught in onVlanInterfaceOperationalStateDown", e);
+        LOG.error("Exception observed in handling deletion of VPN Interface {}. ", intrf.getName(), e);
       }
     }
 
@@ -127,22 +148,38 @@ public class InterfaceStateChangeListener extends AbstractDataChangeListener<Int
     protected void update(InstanceIdentifier<Interface> identifier,
             Interface original, Interface update) {
       LOG.trace("Operation Interface update event - Old: {}, New: {}", original, update);
+      if(original.getOperStatus().equals(Interface.OperStatus.Unknown) || update.getOperStatus().equals(Interface.OperStatus.Unknown)){
+          LOG.debug("Interface state change is from/to UNKNOWN. Ignoring the update event.");
+          return;
+      }
       String interfaceName = update.getName();
-      org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface
-          intf = InterfaceUtils.getInterface(broker, interfaceName);
-      if (intf != null && intf.getType().equals(Tunnel.class)) {
-        /*
-        // FIXME: Blocked until tunnel event[vxlan/gre] support is available
-        BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(update);
-        if(update.getOperStatus().equals(Interface.OperStatus.Up)) {
-          //advertise all prefixes in all vpns for this dpn to bgp
-          // vpnInterfaceManager.updatePrefixesForDPN(dpnId, VpnInterfaceManager.UpdateRouteAction.ADVERTISE_ROUTE);
-          vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceUp(update);
-        } else if(update.getOperStatus().equals(Interface.OperStatus.Down)) {
-          //withdraw all prefixes in all vpns for this dpn from bgp
-          // vpnInterfaceManager.updatePrefixesForDPN(dpnId, VpnInterfaceManager.UpdateRouteAction.WITHDRAW_ROUTE);
-          vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceDown(update);
-        }*/
+      BigInteger dpId = InterfaceUtils.getDpIdFromInterface(update);
+      if (update != null) {
+          if (update.getType().equals(Tunnel.class)) {
+            /*
+            // FIXME: Blocked until tunnel event[vxlan/gre] support is available
+            BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(update);
+            if(update.getOperStatus().equals(Interface.OperStatus.Up)) {
+              //advertise all prefixes in all vpns for this dpn to bgp
+              // vpnInterfaceManager.updatePrefixesForDPN(dpnId, VpnInterfaceManager.UpdateRouteAction.ADVERTISE_ROUTE);
+              vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceUp(update);
+            } else if(update.getOperStatus().equals(Interface.OperStatus.Down)) {
+              //withdraw all prefixes in all vpns for this dpn from bgp
+              // vpnInterfaceManager.updatePrefixesForDPN(dpnId, VpnInterfaceManager.UpdateRouteAction.WITHDRAW_ROUTE);
+              vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceDown(update);
+            }*/
+          } else {
+              if (update.getOperStatus().equals(Interface.OperStatus.Up)) {
+                  vpnInterfaceManager.processVpnInterfaceUp(dpId, interfaceName, update.getIfIndex(), true);
+                  vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceUp(update);
+              } else if (update.getOperStatus().equals(Interface.OperStatus.Down)) {
+                  if (VpnUtil.isVpnInterfaceConfigured(broker, interfaceName)) {
+                      vpnInterfaceManager.processVpnInterfaceDown(dpId, interfaceName, update.getIfIndex(), true,
+                              false);
+                      vpnInterfaceManager.getVpnSubnetRouteHandler().onInterfaceDown(update);
+                  }
+              }
+          }
       }
 
     }
@@ -165,7 +202,7 @@ public class InterfaceStateChangeListener extends AbstractDataChangeListener<Int
             RouterInterface routerInterface = optRouterInterface.get();
             String routerName = routerInterface.getRouterName();
             LOG.debug("Handling DOWN event for router interface {} in Router {}", interfaceName, routerName);
-            vpnInterfaceManager.removeFromNeutronRouterDpnsMap(routerName, interfaceName, dpnId);
+            vpnInterfaceManager.removeFromNeutronRouterDpnsMap(routerName, interfaceName,dpnId);
         } else {
             LOG.debug("No Router interface configured to handle  DOWN event for {}", interfaceName);
         }
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/MacEntry.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/MacEntry.java
new file mode 100644 (file)
index 0000000..9bf4a55
--- /dev/null
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2015 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager;
+
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
+
+import java.net.InetAddress;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.TimeUnit;
+
+public class MacEntry implements Delayed {
+
+    private long expiryTime;
+    private String vpnName;
+    private MacAddress macAddress;
+    private InetAddress ipAddress;
+    private String interfaceName;
+
+    public MacEntry(long delay, String vpnName, MacAddress macAddress, InetAddress inetAddress, String interfaceName) {
+        this.expiryTime = System.currentTimeMillis() + delay;
+        this.vpnName = vpnName;
+        this.macAddress = macAddress;
+        this.ipAddress = inetAddress;
+        this.interfaceName = interfaceName;
+    }
+
+    public String getVpnName() {
+        return vpnName;
+    }
+
+    public void setVpnName(String vpnName) {
+        this.vpnName = vpnName;
+    }
+
+
+    @Override
+    public String toString() {
+        return "MacEntry [expiryTime=" + expiryTime + ", vpnName=" + vpnName + ", macAddress=" + macAddress
+              + ", ipAddress=" + ipAddress + ", interfaceName=" + interfaceName + "]";
+    }
+
+    public MacAddress getMacAddress() {
+        return macAddress;
+    }
+
+    public String getInterfaceName() {
+        return interfaceName;
+    }
+
+    public void setInterfaceName(String interfaceName) {
+        this.interfaceName = interfaceName;
+    }
+
+    public InetAddress getIpAddress() {
+        return ipAddress;
+    }
+
+
+    @Override
+    public int compareTo(Delayed obj) {
+        if (this.expiryTime > ((MacEntry) obj).expiryTime) {
+            return -1;
+        } else if (this.expiryTime < ((MacEntry) obj).expiryTime) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public long getDelay(TimeUnit arg0) {
+        long diff = expiryTime - System.currentTimeMillis();
+        return diff;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result
+                + ((macAddress == null) ? 0 : macAddress.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        boolean result = false;
+        if (getClass() != obj.getClass())
+            return result;
+        else{
+        MacEntry other = (MacEntry) obj;
+        result = vpnName.equals(other.vpnName) && macAddress.equals(other.macAddress) && ipAddress.equals(other.ipAddress) && interfaceName.equals(other.interfaceName);
+        }
+        return result;
+    }
+
+
+}
index 1a6128039877fd56c4bcb09274ded39a606e8682..0d52c40456c2dd204624130576ef83f34b6374ad 100644 (file)
@@ -60,7 +60,7 @@ public class SubnetOpDpnManager {
                 return null;
             }
             SubnetToDpnBuilder subDpnBuilder = new SubnetToDpnBuilder().setKey(new SubnetToDpnKey(dpnId));
-            List<VpnInterfaces> vpnIntfList = new ArrayList<>();
+            List<VpnInterfaces> vpnIntfList = new ArrayList<VpnInterfaces>();
             subDpnBuilder.setVpnInterfaces(vpnIntfList);
             subDpn = subDpnBuilder.build();
             logger.trace("Creating SubnetToDpn entry for subnet  " + subnetId.getValue() + " with DPNId "+ dpnId);
@@ -151,8 +151,8 @@ public class SubnetOpDpnManager {
             logger.trace("Creating PortOpData entry for port " + intfName + " with DPNId "+ dpnId);
             MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier, portOpEntry);
         } catch (Exception ex) {
-            logger.error("Addition of Interface " + intfName + " for SubnetToDpn on subnet " +
-                    subnetId.getValue() + " with DPN " + dpnId + " failed {}" + ex);
+          logger.error("Addition of Interface " + intfName + " for SubnetToDpn on subnet " +
+                  subnetId.getValue() + " with DPN " + dpnId + " failed {}" + ex);
         } finally {
         }
     }
@@ -210,7 +210,7 @@ public class SubnetOpDpnManager {
     }
 
     public PortOpDataEntry getPortOpDataEntry(String intfName) {
-        // Remove PortOpData and return out
+     // Remove PortOpData and return out
         InstanceIdentifier<PortOpDataEntry> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).
                 child(PortOpDataEntry.class, new PortOpDataEntryKey(intfName)).build();
         Optional<PortOpDataEntry> optionalPortOp = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
index d87e98ad18dd2777806548c9c6ddd98d7ec2c568..d8212dfc66969f85414358a91b33ae972bc72000 100644 (file)
@@ -15,27 +15,22 @@ import org.opendaylight.controller.liblldp.EtherTypes;
 import org.opendaylight.controller.liblldp.NetUtils;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
-import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
-import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
-import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.tag.name.map.ElanTagName;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnIdToVpnInstance;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.*;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.meta.rev160406._if.indexes._interface.map.IfIndexInterface;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIdsKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkMaps;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.networkmaps.NetworkMap;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.networkmaps.NetworkMapKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.tag.name.map.ElanTagName;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.*;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.opendaylight.genius.mdsalutil.packet.ARP;
 import org.opendaylight.genius.mdsalutil.packet.IPv4;
@@ -98,16 +93,34 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
                     byte[] dstIp = Ints.toByteArray(ipv4.getDestinationAddress());
                     String dstIpStr = toStringIpAddress(dstIp);
                     String srcIpStr = toStringIpAddress(srcIp);
-                    // FIXME 7: To be fixed with VPNManager patch
                     /*if (VpnUtil.getNeutronPortNamefromPortFixedIp(broker, dstIpStr) != null) {
                         s_logger.debug("SubnetRoutePacketInHandler: IPv4 Packet received with "
                                 + "Target IP {} is a valid Neutron port, ignoring subnet route processing", dstIpStr);
                         return;
                     }*/
-                    long elanTag = MetaDataUtil.getElanTagFromMetadata(metadata);
+                    long vpnId = MetaDataUtil.getVpnIdFromMetadata(metadata);
+                    s_logger.info("SubnetRoutePacketInHandler: Processing IPv4 Packet received with Source IP {} "
+                            + "and Target IP {} and vpnId {}", srcIpStr, dstIpStr, vpnId);
+
+                    InstanceIdentifier<VpnIds> vpnIdsInstanceIdentifier = getVpnIdToVpnInstanceIdentifier(vpnId);
+                    Optional<VpnIds> vpnIdsOptional = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnIdsInstanceIdentifier);
+                    if(!vpnIdsOptional.isPresent()) {
+                        // Donot trigger subnetroute logic for packets from unknown VPNs
+                        s_logger.info("Ignoring IPv4 packet with destination Ip {} and source Ip {} as it came on unknown VPN with ID {}", dstIpStr, srcIpStr, vpnId);
+                        return;
+                    }
+                    // It is an ARP request on a configured VPN.  So we must attempt to respond.
+                    VpnIds vpnIds = vpnIdsOptional.get();
+                    if (VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnIds.getVpnInstanceName(), dstIpStr) !=
+                            null) {
+                        s_logger.debug("SubnetRoutePacketInHandler: IPv4 Packet received with "
+                                + "Target IP {} is a valid Neutron port, ignoring subnet route processing", dstIpStr);
+                        return;
+                    }
+                    long elanTag = getElanTagFromSubnetRouteMetadata(metadata);
                     if (elanTag == 0) {
                         s_logger.error("SubnetRoutePacketInHandler: elanTag value from metadata found to be 0, for IPv4 " +
-                                "Packet received with Target IP {}", dstIpStr);
+                                " Packet received with Target IP {}", dstIpStr);
                         return;
                     }
                     s_logger.info("SubnetRoutePacketInHandler: Processing IPv4 Packet received with Source IP {} "
@@ -127,8 +140,9 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
             }
             return;
         }
+        //All Arp responses learning for invisble IPs will now be handled by VpnManager
 
-        if (tableId == NwConstants.L3_INTERFACE_TABLE) {
+        /*if (tableId == NwConstants.L3_INTERFACE_TABLE) {
             s_logger.trace("SubnetRoutePacketInHandler: Packet from Table {} received as {}",
                     NwConstants.L3_INTERFACE_TABLE, notification);
             try {
@@ -142,7 +156,7 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
                 if (pkt instanceof ARP) {
                     s_logger.debug("SubnetRoutePacketInHandler: ARP packet received");
                     ARP arpPacket = (ARP) pkt;
-                    boolean arpReply = (arpPacket.getOpCode() == 2);
+                    boolean arpReply = (arpPacket.getOpCode() == 2) ? true : false;
                     if (arpReply) {
                         //Handle subnet routes arp responses
                         s_logger.debug("SubnetRoutePacketInHandler: ARP reply received");
@@ -150,15 +164,34 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
                         byte[] respDst = arpPacket.getTargetProtocolAddress();
                         String respIp = toStringIpAddress(respSrc);
                         String check = toStringIpAddress(respDst) + respIp;
-                        // FIXME 8: To be fixed with VPNManager patch
-                        /*if (VpnUtil.getNeutronPortNamefromPortFixedIp(broker, respIp) != null) {
+                        if (VpnUtil.getNeutronPortNamefromPortFixedIp(broker, respIp) != null) {
                             s_logger.debug("SubnetRoutePacketInHandler: ARP reply Packet received with "
                                     + "Source IP {} which is a valid Neutron port, ignoring subnet route processing", respIp);
                             return;
-                        }*/
+                        }
                         String destination = VpnUtil.getIpPrefix(respIp);
+                        String srcIp = toStringIpAddress(respSrc);
+                        String destIp = toStringIpAddress(respDst);
+                        long vpnId = MetaDataUtil.getVpnIdFromMetadata(metadata);
+                        s_logger.info("SubnetRoutePacketInHandler: Processing ARP response Packet received with Source IP {} "
+                                + "and Target IP {} and vpnId {}", srcIp, destIp, vpnId);
+                        InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds> vpnIdsInstanceIdentifier = getVpnIdToVpnInstanceIdentifier(vpnId);
+                        Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds> vpnIdsOptional = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnIdsInstanceIdentifier);
+                        if(!vpnIdsOptional.isPresent()) {
+                            // Donot trigger subnetroute logic for packets from unknown VPNs
+                            s_logger.info("Ignoring ARP response packet with destination Ip {} and source Ip {} as it came on with VPN ID {}", destIp, srcIp, vpnId);
+                            return;
+                        }
+                        // It is an ARP request on a configured VPN.  So we must attempt to respond.
+                        org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds vpnIds = vpnIdsOptional.get();
+                        if (VpnUtil.getNeutronPortFromVpnPortFixedIp(broker, vpnIds.getVpnInstanceName(), srcIp) != null) {
+                            s_logger.debug("SubnetRoutePacketInHandler: ARP response Packet received with "
+                                    + "Target IP {} is a valid Neutron port, ignoring subnet route processing", destIp);
+                            return;
+                        }
+                        String destination = VpnUtil.getIpPrefix(srcIp);
                         long portTag = MetaDataUtil.getLportFromMetadata(metadata).intValue();
-                        s_logger.info("SubnetRoutePacketInHandler: ARP reply received for target IP {} from LPort {}" + respIp, portTag);
+                        s_logger.info("SubnetRoutePacketInHandler: ARP reply received for target IP {} from LPort {}" + srcIp, portTag);
                         IfIndexInterface interfaceInfo = VpnUtil.getInterfaceInfoByInterfaceTag(broker, portTag);
                         String ifName = interfaceInfo.getInterfaceName();
                         InstanceIdentifier<VpnInterface> vpnIfIdentifier = VpnUtil.getVpnInterfaceIdentifier(ifName);
@@ -186,9 +219,12 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
                                                     VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnInterface.getVpnInstanceName(), destination));
                                     String nextHopIp = nextHopIpAddr.split("/")[0];
                                     // FIXME 9: To be fixed with VPNManager patch
-                                    /*Adjacency newAdj = new AdjacencyBuilder().setIpAddress(destination).setKey
+                                    /Adjacency newAdj = new AdjacencyBuilder().setIpAddress(destination).setKey
                                             (new AdjacencyKey(destination)).setNextHopIp(nextHopIp).build();
-                                    adjacencyList.add(newAdj);*/
+                                    adjacencyList.add(newAdj);
+                                    Adjacency newAdj = new AdjacencyBuilder().setIpAddress(destination).setKey
+                                            (new AdjacencyKey(destination)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
+                                    adjacencyList.add(newAdj);
                                     Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
                                     VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface.getName())).
                                             setName(vpnInterface.getName()).setVpnInstanceName(vpnInterface.getVpnInstanceName()).
@@ -205,7 +241,7 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
                 s_logger.error("SubnetRoutePacketInHandler: Failed to handle subnetroute Table " + NwConstants.L3_INTERFACE_TABLE +
                         " packets ", ex);
             }
-        }
+        }*/
     }
 
     private static BigInteger getTargetDpnForPacketOut(DataBroker broker, long elanTag, int ipAddress) {
@@ -276,6 +312,10 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
         return Long.parseLong(dpId);
     }
 
+    public static long getElanTagFromSubnetRouteMetadata(BigInteger metadata) {
+        return ((metadata.and(MetaDataUtil.METADATA_MASK_ELAN_SUBNET_ROUTE)).shiftRight(32)).longValue();
+    }
+
     private void sendArpRequest(BigInteger dpnId, long groupId, byte[] abySenderMAC, byte[] abySenderIpAddress,
                                 byte[] abyTargetIpAddress) {
 
@@ -291,7 +331,7 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
             arpPacket = createARPPacket(ARP.REQUEST, abySenderMAC, abySenderIpAddress, VpnConstants.MAC_Broadcast,
                     abyTargetIpAddress);
             ethPacket = createEthernetPacket(abySenderMAC, VpnConstants.EthernetDestination_Broadcast, arpPacket);
-            lstActionInfo = new ArrayList<>();
+            lstActionInfo = new ArrayList<ActionInfo>();
             lstActionInfo.add(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }));
             transmitPacketInput = MDSALUtil.getPacketOutDefault(lstActionInfo, ethPacket, dpnId);
             packetService.transmitPacket(transmitPacketInput);
@@ -301,7 +341,7 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
     }
 
     private static byte[] createARPPacket(short opCode, byte[] senderMacAddress, byte[] senderIP, byte[] targetMacAddress,
-                                          byte[] targetIP) {
+                                         byte[] targetIP) {
         ARP arp = new ARP();
         byte[] rawArpPkt = null;
         try {
@@ -338,4 +378,11 @@ public class SubnetRoutePacketInHandler implements PacketProcessingListener {
         }
         return rawEthPkt;
     }
+
+    static InstanceIdentifier<VpnIds>
+    getVpnIdToVpnInstanceIdentifier(long vpnId) {
+        return InstanceIdentifier.builder(VpnIdToVpnInstance.class)
+                .child(VpnIds.class,
+                        new VpnIdsKey(Long.valueOf(vpnId))).build();
+    }
 }
index bb5718c38796387e9213518f02784cf5f4297ae9..f3c2a986c26f25f25e488964bdd9c2ff5c7f8f57 100644 (file)
@@ -12,10 +12,11 @@ import java.math.BigInteger;
 
 public class VpnConstants {
     public static final String VPN_IDPOOL_NAME = "vpnservices";
-    public static final long VPN_IDPOOL_START = 100L;
+    public static final long VPN_IDPOOL_START = 70000L;
     public static final String VPN_IDPOOL_SIZE = "100000";
     public static final short DEFAULT_FLOW_PRIORITY = 10;
-    public static final short L3VPN_SERVICE_IDENTIFIER = 3;
+    public static final int DEFAULT_LPORT_DISPATCHER_FLOW_PRIORITY = 1;
+    public static final short L3VPN_SERVICE_IDENTIFIER = 2;
     public static final long INVALID_ID = -1;
     public static final String SEPARATOR = ".";
     public static final BigInteger COOKIE_VM_INGRESS_TABLE = new BigInteger("8000001", 16);
@@ -26,9 +27,16 @@ public class VpnConstants {
     public static final long PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS = 10000;
     public static final int ELAN_GID_MIN = 200000;
     public static final short ELAN_SMAC_TABLE = 50;
+    public static final short LPORT_DISPATCHER_TABLE = 17;
     public static final short FIB_TABLE = 21;
     public static byte[] EthernetDestination_Broadcast = new byte[] { (byte) 0xFF, (byte) 0xFF, (byte) 0xFF,
             (byte) 0xFF, (byte) 0xFF, (byte) 0xFF };
     public static byte[] MAC_Broadcast = new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0 };
+    public enum ITMTunnelLocType {
+        Invalid(0), Internal(1), External(2), Hwvtep(3);
 
+        private final int type;
+        ITMTunnelLocType(int id) { this.type = id; }
+        public int getValue() { return type; }
+    }
 }
index ff4df7c8d28a5a99b646434b7c1c2ac116c1c47c..acdbf84960ead5fc7008c0b22702081138bdecd0 100644 (file)
@@ -7,8 +7,12 @@
  */
 package org.opendaylight.netvirt.vpnmanager;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.common.base.*;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterators;
+import com.google.common.util.concurrent.ListenableFuture;
 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
+import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
 
@@ -19,13 +23,26 @@ import com.google.common.util.concurrent.JdkFutureAdapters;
 import org.opendaylight.controller.md.sal.binding.api.*;
 import org.opendaylight.genius.mdsalutil.*;
 import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.VpnTargets;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
+import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeExternal;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeHwvtep;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeInternal;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRouteBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PrefixToInterface;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.add.dpn.event.AddEventDataBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
@@ -37,9 +54,12 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neu
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.remove.dpn.event.RemoveEventDataBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.IpAddresses;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
@@ -61,19 +81,18 @@ import java.util.List;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
-import com.google.common.base.Optional;
-
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 import org.opendaylight.yangtools.concepts.ListenerRegistration;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
@@ -83,7 +102,6 @@ import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
@@ -97,14 +115,12 @@ import org.slf4j.LoggerFactory;
 
 public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface> implements AutoCloseable {
     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
-    private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration;
-    private static final ThreadFactory threadFactory = new ThreadFactoryBuilder()
-        .setNameFormat("NV-VpnIntfMgr-%d").build();
+    private ListenerRegistration<DataChangeListener> listenerRegistration, opListenerRegistration, tunnelInterfaceStateListenerRegistration;
     private final DataBroker broker;
     private final IBgpManager bgpManager;
     private IFibManager fibManager;
     private IMdsalApiManager mdsalManager;
-    private OdlInterfaceRpcService interfaceManager;
+    private OdlInterfaceRpcService ifaceMgrRpcService;
     private ItmRpcService itmProvider;
     private IdManagerService idManager;
     private OdlArputilService arpManager;
@@ -113,16 +129,22 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
     private ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<String, Runnable>();
     private ExecutorService executorService = Executors.newSingleThreadExecutor();
     private InterfaceStateChangeListener interfaceListener;
+    private TunnelInterfaceStateListener tunnelInterfaceStateListener;
     private VpnInterfaceOpListener vpnInterfaceOpListener;
     private ArpNotificationHandler arpNotificationHandler;
+    private NotificationPublishService notificationPublishService;
+    private FibRpcService fibService;
     protected enum UpdateRouteAction {
         ADVERTISE_ROUTE, WITHDRAW_ROUTE
     }
+
     /**
      * Responsible for listening to data change related to VPN Interface
      * Bind VPN Service on the interface and informs the BGP service
      *
      * @param db - dataBroker service reference
+     * @param bgpManager Used to advertise routes to the BGP Router
+     * @param notificationService Used to subscribe to notification events
      */
     public VpnInterfaceManager(final DataBroker db, final IBgpManager bgpManager, NotificationService notificationService) {
         super(VpnInterface.class);
@@ -131,9 +153,10 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         interfaceListener = new InterfaceStateChangeListener(db, this);
         vpnInterfaceOpListener = new VpnInterfaceOpListener();
         arpNotificationHandler = new ArpNotificationHandler(this, broker);
-        notificationService.registerNotificationListener(arpNotificationHandler);
         vpnSubnetRouteHandler = new VpnSubnetRouteHandler(broker, bgpManager, this);
+        tunnelInterfaceStateListener = new TunnelInterfaceStateListener(broker, this);
         notificationService.registerNotificationListener(vpnSubnetRouteHandler);
+        notificationService.registerNotificationListener(arpNotificationHandler);
         registerListener(db);
     }
 
@@ -141,15 +164,16 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         this.mdsalManager = mdsalManager;
     }
 
-    public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
-        this.interfaceManager = interfaceManager;
-        interfaceListener.setInterfaceManager(interfaceManager);
+    public void setIfaceMgrRpcService(OdlInterfaceRpcService ifMgrRpcService) {
+        this.ifaceMgrRpcService = ifMgrRpcService;
+        interfaceListener.setIfaceMgrRpcService(ifMgrRpcService);
     }
 
     public void setITMProvider(ItmRpcService itmProvider) {
         this.itmProvider = itmProvider;
     }
 
+
     public void setFibManager(IFibManager fibManager) {
         this.fibManager = fibManager;
     }
@@ -158,6 +182,7 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         return this.fibManager;
     }
 
+
     public void setIdManager(IdManagerService idManager) {
         this.idManager = idManager;
         vpnSubnetRouteHandler.setIdManager(idManager);
@@ -167,8 +192,20 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         this.arpManager = arpManager;
     }
 
+    void setNotificationPublishService(NotificationPublishService notificationPublishService) {
+        this.notificationPublishService = notificationPublishService;
+    }
+
     public void setNeutronvpnManager(NeutronvpnService neuService) { this.neuService = neuService; }
 
+    public void setFibRpcService(FibRpcService fibService) {
+        this.fibService = fibService;
+    }
+
+    public FibRpcService getFibRpcService() {
+        return fibService;
+    }
+
     public VpnSubnetRouteHandler getVpnSubnetRouteHandler() {
         return this.vpnSubnetRouteHandler;
     }
@@ -194,140 +231,240 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                     getWildCardPath(), VpnInterfaceManager.this, DataChangeScope.SUBTREE);
             opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
                                                                    getWildCardPath(), vpnInterfaceOpListener, DataChangeScope.SUBTREE);
+            tunnelInterfaceStateListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
+                    getTunnelInterfaceStateListenerPath(), tunnelInterfaceStateListener, DataChangeScope.SUBTREE);
         } catch (final Exception e) {
             LOG.error("VPN Service DataChange listener registration fail!", e);
             throw new IllegalStateException("VPN Service registration Listener failed.", e);
         }
     }
 
+    private InstanceIdentifier<StateTunnelList> getTunnelInterfaceStateListenerPath() {
+        return InstanceIdentifier.create(TunnelsState.class).child(StateTunnelList.class);
+    }
+
     private InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface> getInterfaceListenerPath() {
         return InstanceIdentifier.create(InterfacesState.class)
             .child(org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.class);
     }
 
     @Override
-    protected void add(final InstanceIdentifier<VpnInterface> identifier,
-            final VpnInterface vpnInterface) {
-        LOG.trace("VPN Interface key: {} , value: {}", identifier, vpnInterface );
-        addInterface(identifier, vpnInterface);
-    }
-
-    private void addInterface(final InstanceIdentifier<VpnInterface> identifier,
-                              final VpnInterface vpnInterface) {
+    public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
         LOG.trace("VPN Interface add event - key: {}, value: {}" ,identifier, vpnInterface );
         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
         String interfaceName = key.getName();
 
         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
-        if (interfaceState != null) {
-            // Interface state is up
-            processVpnInterfaceUp(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex());
+        if(interfaceState != null){
+            BigInteger dpnId = BigInteger.ZERO;
+            try{
+                dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
+            }catch (Exception e){
+                LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. ", interfaceName, e);
+                return;
+            }
+            processVpnInterfaceUp(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName,
+                                  interfaceState.getIfIndex(), false);
         } else {
-            LOG.trace("VPN interfaces are not yet operational.");
+            LOG.info("Handling addition of VPN interface {} skipped as interfaceState is not available", interfaceName);
         }
     }
 
-    protected void processVpnInterfaceUp(BigInteger dpId, String interfaceName, int lPortTag) {
+    protected void processVpnInterfaceUp(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceUp) {
 
         VpnInterface vpnInterface = VpnUtil.getConfiguredVpnInterface(broker, interfaceName);
         if(vpnInterface == null) {
             LOG.info("Unable to process add/up for interface {} as it is not configured", interfaceName);
             return;
         }
-        String vpnName = vpnInterface.getVpnInstanceName();
-        LOG.info("Binding vpn service to interface {} ", interfaceName);
-        long vpnId = VpnUtil.getVpnId(broker, vpnName);
-        if (vpnId == VpnConstants.INVALID_ID) {
-            LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
-            return;
-        }
-        boolean waitForVpnInterfaceOpRemoval = false;
-        int numAdjs = 0;
-        VpnInterface opVpnInterface = null;
-        synchronized (interfaceName.intern()) {
-            opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName());
-            if (opVpnInterface != null ) {
-                String opVpnName = opVpnInterface.getVpnInstanceName();
-                String primaryInterfaceIp = null;
-                if(opVpnName.equals(vpnName)) {
-                    // Please check if the primary VRF Entry does not exist for VPNInterface
-                    // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
-                    // back to back
-                    // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
-                    List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(broker, interfaceName);
-                    if (adjs == null) {
-                        LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained",
-                                interfaceName);
-                        return;
-                    }
-                    numAdjs = adjs.size();
-                    for (Adjacency adj : adjs) {
-                        if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
-                            // FIXME 2: To be fixed with VPNManager patch
-                            //primaryInterfaceIp = adj.getNextHopIp();
-                            break;
+        if (!isInterfaceUp) {
+            String vpnName = vpnInterface.getVpnInstanceName();
+            LOG.info("Binding vpn service to interface {} ", interfaceName);
+            long vpnId = VpnUtil.getVpnId(broker, vpnName);
+            if (vpnId == VpnConstants.INVALID_ID) {
+                LOG.trace("VpnInstance to VPNId mapping is not yet available, bailing out now.");
+                return;
+            }
+            boolean waitForVpnInterfaceOpRemoval = false;
+            int numAdjs = 0;
+            VpnInterface opVpnInterface = null;
+            synchronized (interfaceName.intern()) {
+                opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, vpnInterface.getName());
+                if (opVpnInterface != null ) {
+                    String opVpnName = opVpnInterface.getVpnInstanceName();
+                    String primaryInterfaceIp = null;
+                    if(opVpnName.equals(vpnName)) {
+                        // Please check if the primary VRF Entry does not exist for VPNInterface
+                        // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
+                        // back to back
+                        // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
+                        List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(broker, interfaceName);
+                        if (adjs == null) {
+                            LOG.info("VPN Interface {} addition failed as adjacencies for this vpn interface could not be obtained", interfaceName);
+                            return;
                         }
+                        numAdjs = adjs.size();
+                        for (Adjacency adj : adjs) {
+                            if (adj.getMacAddress() != null && !adj.getMacAddress().isEmpty()) {
+                                primaryInterfaceIp = adj.getIpAddress();
+                                break;
+                            }
+                        }
+                        if (primaryInterfaceIp == null) {
+                            LOG.info("VPN Interface {} addition failed as primary adjacency for this vpn interface could not be obtained", interfaceName);
+                            return;
+                        }
+                        // Get the rd of the vpn instance
+                        String rd = getRouteDistinguisher(opVpnName);
+                        VrfEntry vrf = VpnUtil.getVrfEntry(broker, rd, primaryInterfaceIp);
+                            if (vrf != null) {
+                                LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
+                                return;
+                            }
+                        waitForVpnInterfaceOpRemoval = true;
+                    } else {
+                        LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}",
+                             interfaceName, vpnName, opVpnName);
                     }
-                    if (primaryInterfaceIp == null) {
-                        LOG.info("VPN Interface {} addition failed as primary adjacency for this vpn interface could not be obtained",
-                                interfaceName);
-                        return;
-                    }
-                    // Get the rd of the vpn instance
-                    String rd = getRouteDistinguisher(opVpnName);
-                    VrfEntry vrf = VpnUtil.getVrfEntry(broker, rd, primaryInterfaceIp);
-                    if (vrf != null) {
-                        LOG.info("VPN Interface {} already provisioned , bailing out from here.", interfaceName);
-                        return;
+                }
+                if (!waitForVpnInterfaceOpRemoval) {
+                    // Add the VPNInterface and quit
+                    bindService(dpId, vpnName, interfaceName, lPortTag);
+                    updateDpnDbs(dpId, vpnName, interfaceName, true);
+                    processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
+                        vpnInterface);
+                    return;
+                }
+            }
+
+            // FIB didn't get a chance yet to clean up this VPNInterface
+            // Let us give it a chance here !
+            LOG.info("VPN Interface {} waiting for FIB to clean up! ", interfaceName);
+            try {
+                Runnable notifyTask = new VpnNotifyTask();
+                vpnIntfMap.put(interfaceName, notifyTask);
+                synchronized (notifyTask) {
+                    try {
+                        notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
+                    } catch (InterruptedException e) {
                     }
-                    waitForVpnInterfaceOpRemoval = true;
-                } else {
-                    LOG.info("vpn interface {} to go to configured vpn {}, but in operational vpn {}", interfaceName, vpnName, opVpnName);
                 }
+            } finally {
+                vpnIntfMap.remove(interfaceName);
+            }
+
+            opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
+            if (opVpnInterface != null) {
+                LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
+                return;
             }
-            if (!waitForVpnInterfaceOpRemoval) {
-                // Add the VPNInterface and quit
+            // VPNInterface got removed, proceed with Add
+            synchronized (interfaceName.intern()) {
                 bindService(dpId, vpnName, interfaceName, lPortTag);
                 updateDpnDbs(dpId, vpnName, interfaceName, true);
-                processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
+                processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
+                                               vpnInterface);
+            }
+        } else {
+            synchronized (interfaceName.intern()) {
+                // Interface is retained in the DPN, but its Link Up.
+                // Advertise prefixes again for this interface to BGP
+                advertiseAdjacenciesForVpnToBgp(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()),
+                        vpnInterface);
+            }
+        }
+    }
+
+    private void advertiseAdjacenciesForVpnToBgp(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
+        //Read NextHops
+        InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
+        Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
+
+        String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
+        if (rd == null) {
+            LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
+                    intf.getName(), intf.getVpnInstanceName());
+            return;
+        } else {
+            if (rd.equals(intf.getVpnInstanceName())) {
+                LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} as it is in " +
+                        "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
+
                 return;
             }
         }
+        LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} in vpn {} with rd {} ", intf.getName(),
+                intf.getVpnInstanceName(), rd);
 
-        // FIB didn't get a chance yet to clean up this VPNInterface
-        // Let us give it a chance here !
-        LOG.info("VPN Interface {} waiting for FIB to clean up! ", interfaceName);
-        try {
-            Runnable notifyTask = new VpnNotifyTask();
-            vpnIntfMap.put(interfaceName, notifyTask);
-            synchronized (notifyTask) {
-                try {
-                    notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
-                } catch (InterruptedException e) {
+        String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
+        if (nextHopIp == null){
+            LOG.trace("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} is null, returning", intf.getName());
+            return;
+        }
+
+        if (adjacencies.isPresent()) {
+            List<Adjacency> nextHops = adjacencies.get().getAdjacency();
+
+            if (!nextHops.isEmpty()) {
+                LOG.trace("NextHops are " + nextHops);
+                for (Adjacency nextHop : nextHops) {
+                    long label = nextHop.getLabel();
+                    try {
+                        LOG.info("VPN ADVERTISE: Adding Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
+                        bgpManager.advertisePrefix(rd, nextHop.getIpAddress(), nextHopIp, (int)label);
+                        LOG.info("VPN ADVERTISE: Added Fib Entry rd {} prefix {} nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
+                    } catch(Exception e) {
+                        LOG.error("Failed to advertise prefix {} in vpn {} with rd {} for interface {} ",
+                                nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
+                    }
                 }
             }
-        } finally {
-            vpnIntfMap.remove(interfaceName);
         }
+    }
+
+    private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
+        //Read NextHops
+        InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
+        Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
 
-        opVpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
-        if (opVpnInterface != null) {
-            LOG.error("VPN Interface {} removal by FIB did not complete on time, bailing addition ...", interfaceName);
+        String rd = VpnUtil.getVpnRd(broker, intf.getVpnInstanceName());
+        if (rd == null) {
+            LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
+                    intf.getName(), intf.getVpnInstanceName());
             return;
+        } else {
+            if (rd.equals(intf.getVpnInstanceName())) {
+                LOG.info("withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in " +
+                        "internal vpn{} with rd {}", intf.getName(), intf.getVpnInstanceName(), rd);
+                return;
+            }
         }
-        // VPNInterface got removed, proceed with Add
-        synchronized (interfaceName.intern()) {
-            bindService(dpId, vpnName, interfaceName, lPortTag);
-            updateDpnDbs(dpId, vpnName, interfaceName, true);
-            processVpnInterfaceAdjacencies(dpId, VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
+        LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", intf.getName(),
+                intf.getVpnInstanceName(), rd);
+        if (adjacencies.isPresent()) {
+            List<Adjacency> nextHops = adjacencies.get().getAdjacency();
+
+            if (!nextHops.isEmpty()) {
+                LOG.trace("NextHops are " + nextHops);
+                for (Adjacency nextHop : nextHops) {
+                    try {
+                        LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
+                        bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
+                        LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, nextHop.getIpAddress());
+                    } catch(Exception e) {
+                        LOG.error("Failed to withdraw prefix {} in vpn {} with rd {} for interface {} ",
+                                nextHop.getIpAddress(), intf.getVpnInstanceName(), rd, intf.getName(), e);
+                    }
+                }
+            }
         }
     }
 
     private void updateDpnDbs(BigInteger dpId, String vpnName, String interfaceName, boolean add) {
         long vpnId = VpnUtil.getVpnId(broker, vpnName);
         if (dpId == null) {
-            dpId = InterfaceUtils.getDpnForInterface(interfaceManager, interfaceName);
+            dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, interfaceName);
         }
         if(!dpId.equals(BigInteger.ZERO)) {
             if(add)
@@ -343,7 +480,7 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
 
         int instructionKey = 0;
-        List<Instruction> instructions = new ArrayList<>();
+        List<Instruction> instructions = new ArrayList<Instruction>();
 
         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID, ++instructionKey));
         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.L3_FIB_TABLE, ++instructionKey));
@@ -362,9 +499,9 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
     }
 
-    private void processVpnInterfaceAdjacencies(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
+    private void processVpnInterfaceAdjacencies(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier,
+                                                VpnInterface intf) {
         String intfName = intf.getName();
-
         synchronized (intfName) {
             // Read NextHops
             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
@@ -380,18 +517,20 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
                 if (nextHopIp == null){
                     LOG.error("NextHop for interface {} is null", intfName);
+                    return;
                 }
-
-                LOG.trace("NextHops are {}", nextHops);
+                List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(intf.getVpnInstanceName());
+                LOG.trace("NextHops for interface {} are {}", intfName, nextHops);
                 for (Adjacency nextHop : nextHops) {
                     String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
                     long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil
                             .getNextHopLabelKey((rd == null) ? intf.getVpnInstanceName() : rd, prefix));
-                    // FIXME 3: To be fixed with VPNManager patch
-                    /*String adjNextHop = nextHop.getNextHopIp();
-                    value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIp((adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : nextHopIp)
-                            .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());*/
-                    if(nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
+                    List<String> adjNextHop = nextHop.getNextHopIpList();
+                    value.add(new AdjacencyBuilder(nextHop).setLabel(label).setNextHopIpList(
+                            (adjNextHop != null && !adjNextHop.isEmpty()) ? adjNextHop : Arrays.asList(nextHopIp))
+                            .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
+                    if (nextHop.getMacAddress() != null && !nextHop.getMacAddress().isEmpty()) {
+                        LOG.trace("Adding prefix {} to interface {} for vpn {}", prefix, intfName, intf.getVpnInstanceName());
                         VpnUtil.syncUpdate(
                                 broker,
                                 LogicalDatastoreType.OPERATIONAL,
@@ -401,39 +540,190 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                     } else {
                         //Extra route adjacency
                         // FIXME 4: To be fixed with VPNManager patch
-                        /*VpnUtil.syncUpdate(
+                        LOG.trace("Adding prefix {} and nexthopList {} as extra-route for vpn", nextHop.getIpAddress(), nextHop.getNextHopIpList(), intf.getVpnInstanceName() );
+                        VpnUtil.syncUpdate(
                                 broker,
                                 LogicalDatastoreType.OPERATIONAL,
                                 VpnUtil.getVpnToExtrarouteIdentifier(
                                         (rd != null) ? rd : intf.getVpnInstanceName(), nextHop.getIpAddress()),
-                                VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIp()));*/
-
+                                VpnUtil.getVpnToExtraroute(nextHop.getIpAddress(), nextHop.getNextHopIpList()));
                     }
+
                 }
 
                 Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(value);
-                VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug);
+                VpnInterface opInterface = VpnUtil.getVpnInterface(intfName, intf.getVpnInstanceName(), aug, dpnId, Boolean.FALSE);
                 InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(intfName);
                 VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface);
+                long vpnId = VpnUtil.getVpnId(broker, intf.getVpnInstanceName());
                 for (Adjacency nextHop : aug.getAdjacency()) {
                     long label = nextHop.getLabel();
-                    //String adjNextHop = nextHop.getNextHopIp();
+                    List<String> nextHopList = new ArrayList<>(nextHop.getNextHopIpList());
                     if (rd != null) {
-                        addPrefixToBGP(rd, nextHop.getIpAddress(),
-                                            nextHopIp, label);
+                        addToLabelMapper(label, dpnId, nextHop.getIpAddress(), nextHopList, vpnId,
+                                intfName, null,false, rd);
+                        addPrefixToBGP(rd, nextHop.getIpAddress(), nextHopIp, label);
+                        //TODO: ERT - check for VPNs importing my route
+                        for (VpnInstance vpn : vpnsToImportRoute) {
+                            String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
+                            if (vpnRd != null) {
+                                LOG.debug("Exporting route with rd {} prefix {} nexthop {} label {} to VPN {}", vpnRd, nextHop.getIpAddress(), nextHopIp, label, vpn);
+                                VpnUtil.addFibEntryToDS(broker, vpnRd, nextHop.getIpAddress(), nextHopIp, (int) label, RouteOrigin.SELF_IMPORTED);
+                            }
+                        }
                     } else {
                         // ### add FIB route directly
-                        addFibEntryToDS(intf.getVpnInstanceName(), nextHop.getIpAddress(),
-                                            nextHopIp, (int) label);
+                        VpnUtil.addFibEntryToDS(broker, intf.getVpnInstanceName(), nextHop.getIpAddress(), nextHopIp,
+                                                (int) label, RouteOrigin.STATIC);
                     }
                 }
             }
         }
     }
 
+    private List<VpnInstance> getVpnsImportingMyRoute(final String vpnName) {
+        List<VpnInstance> vpnsToImportRoute = new ArrayList<>();
+
+        InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
+                .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
+        Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+        final VpnInstance vpnInstance;
+        if (optVpnInstance.isPresent()) {
+            vpnInstance = optVpnInstance.get();
+        } else {
+            LOG.debug("Could not retrieve vpn instance {} to check for vpns importing the routes", vpnName);
+            return vpnsToImportRoute;
+        }
+
+        Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
+            @Override
+            public boolean apply(VpnInstance input) {
+                return !input.getVpnInstanceName().equals(vpnName);
+            }
+        };
+
+        Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
+            @Override
+            public boolean apply(VpnInstance input) {
+                Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ExportExtcommunity),
+                        getRts(input, VpnTarget.VrfRTType.ImportExtcommunity));
+                return Iterators.size(commonRTs.iterator()) > 0;
+            }
+        };
+
+        Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
+            @Override
+            public String apply(VpnInstance vpnInstance) {
+                //return vpnInstance.getVpnInstanceName();
+                return vpnInstance.getIpv4Family().getRouteDistinguisher();
+            }
+        };
+
+        vpnsToImportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
+                filter(excludeVpn).
+                filter(matchRTs).toList();
+        return vpnsToImportRoute;
+    }
+
+    private List<VpnInstance> getVpnsExportingMyRoute(final String vpnName) {
+        List<VpnInstance> vpnsToExportRoute = new ArrayList<>();
+
+        InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
+                .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
+        Optional<VpnInstance> optVpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+        final VpnInstance vpnInstance;
+        if (optVpnInstance.isPresent()) {
+            vpnInstance = optVpnInstance.get();
+        } else {
+            LOG.debug("Could not retrieve vpn instance {} to check for vpns exporting the routes", vpnName);
+            return vpnsToExportRoute;
+        }
+
+        Predicate<VpnInstance> excludeVpn = new Predicate<VpnInstance>() {
+            @Override
+            public boolean apply(VpnInstance input) {
+                return !input.getVpnInstanceName().equals(vpnName);
+            }
+        };
+
+        Predicate<VpnInstance> matchRTs = new Predicate<VpnInstance>() {
+            @Override
+            public boolean apply(VpnInstance input) {
+                Iterable<String> commonRTs = intersection(getRts(vpnInstance, VpnTarget.VrfRTType.ImportExtcommunity),
+                        getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
+                return Iterators.size(commonRTs.iterator()) > 0;
+            }
+        };
+
+        Function<VpnInstance, String> toInstanceName = new Function<VpnInstance, String>() {
+            @Override
+            public String apply(VpnInstance vpnInstance) {
+                return vpnInstance.getVpnInstanceName();
+            }
+        };
+
+        vpnsToExportRoute = FluentIterable.from(VpnUtil.getAllVpnInstance(broker)).
+                filter(excludeVpn).
+                filter(matchRTs).toList();
+        return vpnsToExportRoute;
+    }
+
+    private <T> Iterable<T> intersection(final Collection<T> collection1, final Collection<T> collection2) {
+        final Predicate<T> inPredicate = Predicates.<T>in(collection2);
+        return new Iterable<T>() {
+            @Override
+            public Iterator<T> iterator() {
+                return Iterators.filter(collection1.iterator(), inPredicate);
+            }
+        };
+    }
+
+    private List<String> getRts(VpnInstance vpnInstance, VpnTarget.VrfRTType rtType) {
+        String name = vpnInstance.getVpnInstanceName();
+        List<String> rts = new ArrayList<>();
+        VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
+        if (vpnConfig == null) {
+            LOG.trace("vpn config is not available for {}", name);
+            return rts;
+        }
+        VpnTargets targets = vpnConfig.getVpnTargets();
+        if (targets == null) {
+            LOG.trace("vpn targets not available for {}", name);
+            return rts;
+        }
+        List<VpnTarget> vpnTargets = targets.getVpnTarget();
+        if (vpnTargets == null) {
+            LOG.trace("vpnTarget values not available for {}", name);
+            return rts;
+        }
+        for (VpnTarget target : vpnTargets) {
+            //TODO: Check for RT type is Both
+            if(target.getVrfRTType().equals(rtType) ||
+                    target.getVrfRTType().equals(VpnTarget.VrfRTType.Both)) {
+                String rtValue = target.getVrfRTValue();
+                rts.add(rtValue);
+            }
+        }
+        return rts;
+    }
+
+    private List<String> getExportRts(VpnInstance vpnInstance) {
+        List<String> exportRts = new ArrayList<>();
+        VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
+        VpnTargets targets = vpnConfig.getVpnTargets();
+        List<VpnTarget> vpnTargets = targets.getVpnTarget();
+        for (VpnTarget target : vpnTargets) {
+            if (target.getVrfRTType().equals(VpnTarget.VrfRTType.ExportExtcommunity)) {
+                String rtValue = target.getVrfRTValue();
+                exportRts.add(rtValue);
+            }
+        }
+        return exportRts;
+    }
+
     private void makeArpFlow(BigInteger dpId,short sIndex, int lPortTag, String vpnInterfaceName,
                              long vpnId, ArpReplyOrRequest replyOrRequest, int addOrRemoveFlow){
-        List<MatchInfo> matches = new ArrayList<>();
+        List<MatchInfo> matches = new ArrayList<MatchInfo>();
         BigInteger metadata = MetaDataUtil.getMetaDataForLPortDispatcher(lPortTag, ++sIndex, BigInteger.valueOf(vpnId));
         BigInteger metadataMask = MetaDataUtil.getMetaDataMaskForLPortDispatcher(MetaDataUtil.METADATA_MASK_SERVICE_INDEX,
                 MetaDataUtil.METADATA_MASK_LPORT_TAG, MetaDataUtil.METADATA_MASK_VRFID);
@@ -446,9 +736,12 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         matches.add(new MatchInfo(MatchFieldType.arp_op, new long[] { replyOrRequest.getArpOperation() }));
 
         // Instruction to punt to controller
-        List<InstructionInfo> instructions = new ArrayList<>();
-        List<ActionInfo> actionsInfos = new ArrayList<>();
+        List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
+        List<ActionInfo> actionsInfos = new ArrayList<ActionInfo>();
         actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[] {}));
+        actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{
+                Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)}));
+
         instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos));
 
         // Install the flow entry in L3_INTERFACE_TABLE
@@ -491,16 +784,16 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
         if (dpnInVpn.isPresent()) {
             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id.child(
-                org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance
-                    .op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
+                    org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                            .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
                     new VpnInterfacesKey(intfName)), vpnInterface);
         } else {
             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
                                     VpnUtil.getVpnInstanceOpDataIdentifier(rd),
-                                    VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId));
+                                    VpnUtil.getVpnInstanceOpDataBuilder(rd, vpnId, vpnName));
             VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
-            List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces =  new ArrayList<>();
+            List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                    .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces =  new ArrayList<>();
             vpnInterfaces.add(vpnInterface);
             VpnUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, id,
                               vpnToDpnList.setVpnInterfaces(vpnInterfaces).build());
@@ -517,6 +810,46 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                 itmProvider.buildTunnelFromDPNToDCGW(dpnId, new IpAddress(dcGW.toCharArray()));
             }*/
             fibManager.populateFibOnNewDpn(dpnId, vpnId, (rd == null) ? vpnName : rd);
+            publishAddNotification(dpnId, vpnName, rd);
+            //TODO: IRT - import local routes to vpn. check for the VPNs exporting the routes
+            //FIXME: do we need to handle here. since already routes are imported to this vpn
+
+        }
+    }
+
+    void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
+        List<VpnInstance> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
+        long vpnId = VpnUtil.getVpnId(broker, vpnName);
+        for (VpnInstance vpn : vpnsToExportRoute) {
+            String rd = vpn.getIpv4Family().getRouteDistinguisher();
+            long exportingVpnId = VpnUtil.getVpnId(broker, vpn.getVpnInstanceName());
+            List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(broker, vpn.getIpv4Family().getRouteDistinguisher());
+            if (vrfEntries != null) {
+                for (VrfEntry vrfEntry : vrfEntries) {
+                    try {
+                        if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) {
+                            continue;
+                        }
+                        String prefix = vrfEntry.getDestPrefix();
+                        long label = vrfEntry.getLabel();
+                        List<String> nextHops = vrfEntry.getNextHopAddressList();
+                        SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
+                        for (String nh : nextHops) {
+                            if (route != null) {
+                                LOG.info("Importing subnet route fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
+                                importSubnetRouteForNewVpn(rd, prefix, nh, (int)label, route);
+                            } else {
+                                LOG.info("Importing fib entry rd {} prefix {} nexthop {} label {} to vpn {}", vpnRd, prefix, nh, label, vpn.getVpnInstanceName());
+                                VpnUtil.addFibEntryToDS(broker, vpnRd, prefix, nh, (int)label, RouteOrigin.SELF_IMPORTED);
+                            }
+                        }
+                    } catch (Exception e) {
+                        LOG.error("Exception occurred while importing route with prefix {} label {} nexthop {} from vpn {} to vpn {}", vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), vpn.getVpnInstanceName(), vpnName);
+                    }
+                }
+            } else {
+                LOG.info("No vrf entries to import from vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getIpv4Family().getRouteDistinguisher());
+            }
         }
     }
 
@@ -526,19 +859,26 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         InstanceIdentifier<VpnToDpnList> id = VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
         Optional<VpnToDpnList> dpnInVpn = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
         if (dpnInVpn.isPresent()) {
-            List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
+            List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                    .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
             org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces
                     currVpnInterface = new VpnInterfacesBuilder().setInterfaceName(intfName).build();
 
             if (vpnInterfaces.remove(currVpnInterface)) {
                 if (vpnInterfaces.isEmpty()) {
-                    VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
-                    fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
+                    List<IpAddresses> ipAddresses = dpnInVpn.get().getIpAddresses();
+                    if (ipAddresses == null || ipAddresses.isEmpty()) {
+                        LOG.debug("Sending cleanup event for dpn {} in VPN {}", dpnId, vpnName);
+                        VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id, VpnUtil.DEFAULT_CALLBACK);
+                        fibManager.cleanUpDpnForVpn(dpnId, vpnId, rd);
+                        publishRemoveNotification(dpnId, vpnName, rd);
+                    } else {
+                        LOG.debug("vpn interfaces are empty but ip addresses are present for the vpn {} in dpn {}", vpnName, dpnId);
+                    }
                 } else {
                     VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL, id.child(
-                        org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                            .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
+                        org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                                .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
                             new VpnInterfacesKey(intfName)), VpnUtil.DEFAULT_CALLBACK);
                 }
             }
@@ -547,8 +887,9 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
     private void addPrefixToBGP(String rd, String prefix, String nextHopIp, long label) {
         try {
-            //FIXME: TBD once odl-fib yang has nexthoplist and related changes follow
-            //bgpManager.addPrefix(rd, prefix, nextHopIp, (int)label);
+            LOG.info("ADD: Adding Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
+            bgpManager.addPrefix(rd, prefix, Arrays.asList(nextHopIp), (int)label, RouteOrigin.STATIC);
+            LOG.info("ADD: Added Fib entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHopIp, label);
         } catch(Exception e) {
             LOG.error("Add prefix failed", e);
         }
@@ -560,7 +901,7 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
     }
 
     @Override
-    protected void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
+    public void remove( InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
         LOG.trace("Remove event - key: {}, value: {}" ,identifier, vpnInterface );
         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
         String interfaceName = key.getName();
@@ -569,48 +910,81 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         Optional<VpnInterface> existingVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface interfaceState =
             InterfaceUtils.getInterfaceStateFromOperDS(broker, interfaceName);
-
-        if (existingVpnInterface.isPresent() && interfaceState != null) {
-            processVpnInterfaceDown(InterfaceUtils.getDpIdFromInterface(interfaceState), interfaceName, interfaceState.getIfIndex(), false);
-        } else {
+        if(existingVpnInterface.isPresent()){
+            BigInteger dpnId = BigInteger.ZERO;
+            Boolean dpnIdRetrieved = Boolean.FALSE;
+            if(interfaceState != null){
+                try{
+                    dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
+                    dpnIdRetrieved = Boolean.TRUE;
+                }catch (Exception e){
+                    LOG.error("Unable to retrieve dpnId from interface operational data store for interface {}. Fetching from vpn interface op data store. ", interfaceName, e);
+                }
+            }
+            if(dpnIdRetrieved == Boolean.FALSE){
+                LOG.info("dpnId for {} has not been retrieved yet. Fetching from vpn interface operational DS", interfaceName);
+                dpnId = existingVpnInterface.get().getDpnId();
+            }
+            processVpnInterfaceDown(dpnId, interfaceName, interfaceState.getIfIndex(), false, true);
+        }else{
             LOG.warn("VPN interface {} was unavailable in operational data store to handle remove event", interfaceName);
         }
     }
 
-    protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown) {
-        VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
-        if(vpnInterface == null) {
-            LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
-            return;
-        }
-        String vpnName = vpnInterface.getVpnInstanceName();
+    protected void processVpnInterfaceDown(BigInteger dpId, String interfaceName, int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval) {
         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
+        if (!isInterfaceStateDown) {
+            synchronized (interfaceName.intern()) {
+                VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
+                if(vpnInterface == null){
+                    LOG.info("Unable to process delete/down for interface {} as it is not available in operational data store", interfaceName);
+                    return;
+                }else{
+                    String vpnName = vpnInterface.getVpnInstanceName();
+                    if(!vpnInterface.isScheduledForRemove()){
+                        VpnUtil.updateVpnInterface(broker, vpnInterface.getName(), vpnInterface.getDpnId(), vpnInterface.getVpnInstanceName(), Boolean.TRUE);
+                        removeAdjacenciesFromVpn(dpId, identifier, vpnInterface);
+                        LOG.info("Unbinding vpn service from interface {} ", interfaceName);
+                        unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown, isConfigRemoval);
+                    }else{
+                        LOG.info("Unbinding vpn service for interface {} has already been scheduled by a different event ", interfaceName);
+                        return;
+                    }
+                }
+            }
 
-        synchronized (interfaceName.intern()) {
-            removeAdjacenciesFromVpn(identifier, vpnInterface);
-            LOG.info("Unbinding vpn service from interface {} ", interfaceName);
-            unbindService(dpId, vpnName, interfaceName, lPortTag, isInterfaceStateDown);
-        }
-
-        // FIB didn't get a chance yet to clean up this VPNInterface
-        // Let us give it a chance here !
-        LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
-        try {
-            Runnable notifyTask = new VpnNotifyTask();
-            vpnIntfMap.put(interfaceName, notifyTask);
-            synchronized (notifyTask) {
-                try {
-                    notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
-                } catch (InterruptedException e) {
+            // FIB didn't get a chance yet to clean up this VPNInterface
+            // Let us give it a chance here !
+            LOG.info("VPN Interface {} removal waiting for FIB to clean up ! ", interfaceName);
+            try {
+                Runnable notifyTask = new VpnNotifyTask();
+                vpnIntfMap.put(interfaceName, notifyTask);
+                synchronized (notifyTask) {
+                    try {
+                        notifyTask.wait(VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS);
+                    } catch (InterruptedException e) {
+                    }
                 }
+            } finally {
+                vpnIntfMap.remove(interfaceName);
             }
-        } finally {
-            vpnIntfMap.remove(interfaceName);
+        } else {
+                synchronized (interfaceName.intern()) {
+                    // Interface is retained in the DPN, but its Link Down.
+                    // Only withdraw the prefixes for this interface from BGP
+                    VpnInterface vpnInterface = VpnUtil.getOperationalVpnInterface(broker, interfaceName);
+                    if(vpnInterface == null){
+                        LOG.info("Unable to withdraw adjacencies for vpn interface {} from BGP as it is not available in operational data store", interfaceName);
+                        return;
+                    }else {
+                        withdrawAdjacenciesForVpnFromBgp(identifier, vpnInterface);
+                    }
+                }
         }
 
     }
 
-    private void removeAdjacenciesFromVpn(final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
+    private void removeAdjacenciesFromVpn(BigInteger dpnId, final InstanceIdentifier<VpnInterface> identifier, VpnInterface intf) {
         //Read NextHops
         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
         Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
@@ -624,20 +998,48 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
             if (!nextHops.isEmpty()) {
                 LOG.trace("NextHops are " + nextHops);
                 for (Adjacency nextHop : nextHops) {
-                    // Commenting the release of ID here as it will be released by FIB
-                   /* VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
-                                      VpnUtil.getNextHopLabelKey(rd, nextHop.getIpAddress()));
-                    VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
-                                   VpnUtil.getPrefixToInterfaceIdentifier(
-                                       VpnUtil.getVpnId(broker, intf.getVpnInstanceName()),
-                                       nextHop.getIpAddress()),
-                                   VpnUtil.DEFAULT_CALLBACK);*/
+                    List<String> nhList = new ArrayList<String>();
+                    if (nextHop.getMacAddress() == null || nextHop.getMacAddress().isEmpty()) {
+                        // This is either an extra-route (or) a learned IP via subnet-route
+                        String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
+                        if (nextHopIp == null || nextHopIp.isEmpty()) {
+                            LOG.error("Unable to obtain nextHopIp for extra-route/learned-route in rd {} prefix {}",
+                                    rd, nextHop.getIpAddress());
+                            continue;
+                        }
+                        nhList = Arrays.asList(nextHopIp);
+                    } else {
+                        // This is a primary adjacency
+                        nhList = nextHop.getNextHopIpList();
+                    }
                     if (rd.equals(intf.getVpnInstanceName())) {
                         //this is an internal vpn - the rd is assigned to the vpn instance name;
                         //remove from FIB directly
-                        removeFibEntryFromDS(intf.getVpnInstanceName(), nextHop.getIpAddress());
+                        for(String nh : nhList) {
+                            VpnUtil.removeFibEntryFromDS(broker, intf.getVpnInstanceName(), nextHop.getIpAddress(), nh);
+                        }
                     } else {
-                        removePrefixFromBGP(rd, nextHop.getIpAddress());
+
+                        List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(intf.getVpnInstanceName());
+                        for (String nh : nextHop.getNextHopIpList()) {
+                            //IRT: remove routes from other vpns importing it
+                            removePrefixFromBGP(rd, nextHop.getIpAddress(), nh);
+                            for (VpnInstance vpn : vpnsToImportRoute) {
+                                String vpnRd = vpn.getIpv4Family().getRouteDistinguisher();
+                                if (vpnRd != null) {
+                                    LOG.info("Removing Exported route with rd {} prefix {} from VPN {}", vpnRd, nextHop.getIpAddress(), vpn.getVpnInstanceName());
+                                    VpnUtil.removeFibEntryFromDS(broker, vpnRd, nextHop.getIpAddress(), nh);
+                                }
+                            }
+                        }
+                    }
+                    String ip = nextHop.getIpAddress().split("/")[0];
+                    VpnPortipToPort vpnPortipToPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(broker,
+                            intf.getVpnInstanceName(), ip);
+                    if (vpnPortipToPort != null && !vpnPortipToPort.isConfig()) {
+                        LOG.trace("VpnInterfaceManager removing adjacency for Interface {} ip {} from VpnPortData Entry",
+                                vpnPortipToPort.getPortName(),ip);
+                        VpnUtil.removeVpnPortFixedIpToPort(broker, intf.getVpnInstanceName(), ip);
                     }
                 }
             }
@@ -646,11 +1048,11 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
 
     private void unbindService(BigInteger dpId, String vpnInstanceName, String vpnInterfaceName,
-                               int lPortTag, boolean isInterfaceStateDown) {
-        if (!isInterfaceStateDown) {
+                               int lPortTag, boolean isInterfaceStateDown, boolean isConfigRemoval) {
+        if (!isInterfaceStateDown && isConfigRemoval) {
             MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION,
-                           InterfaceUtils.buildServiceId(vpnInterfaceName,
-                                                         VpnConstants.L3VPN_SERVICE_IDENTIFIER));
+                    InterfaceUtils.buildServiceId(vpnInterfaceName,
+                            VpnConstants.L3VPN_SERVICE_IDENTIFIER));
         }
         long vpnId = VpnUtil.getVpnId(broker, vpnInstanceName);
         makeArpFlow(dpId, VpnConstants.L3VPN_SERVICE_IDENTIFIER, lPortTag, vpnInterfaceName,
@@ -660,10 +1062,12 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
     }
 
 
-    private void removePrefixFromBGP(String rd, String prefix) {
-        removeFibEntryFromDS(rd, prefix);
+    private void removePrefixFromBGP(String rd, String prefix, String nextHop) {
+        VpnUtil.removeFibEntryFromDS(broker, rd, prefix, nextHop);
         try {
-            bgpManager.withdrawPrefix(rd, prefix);
+            LOG.info("VPN WITHDRAW: Removing Fib Entry rd {} prefix {}", rd, prefix);
+            bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
+            LOG.info("VPN WITHDRAW: Removed Fib Entry rd {} prefix {}", rd, prefix);
         } catch(Exception e) {
             LOG.error("Delete prefix failed", e);
         }
@@ -671,12 +1075,10 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
     @Override
     protected void update(InstanceIdentifier<VpnInterface> identifier, VpnInterface original, VpnInterface update) {
-        if (LOG.isTraceEnabled()) {
-            LOG.trace("Updating VPN Interface : key " + identifier + ",  original value=" + original + ", update " +
-                    "value=" + update);
-        }
+        LOG.trace("Updating VPN Interface : key {},  original value={}, update value={}", identifier, original, update);
         String oldVpnName = original.getVpnInstanceName();
         String newVpnName = update.getVpnInstanceName();
+        BigInteger dpnId = update.getDpnId();
         List<Adjacency> oldAdjs = original.getAugmentation(Adjacencies.class).getAdjacency();
         List<Adjacency> newAdjs = update.getAugmentation(Adjacencies.class).getAdjacency();
         if (oldAdjs == null) {
@@ -698,11 +1100,11 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                     oldAdjs.remove(adj);
                 } else {
                     // add new adjacency - right now only extra route will hit this path
-                    addNewAdjToVpnInterface(identifier, adj);
+                    addNewAdjToVpnInterface(identifier, adj, dpnId);
                 }
             }
             for (Adjacency adj : oldAdjs) {
-                delAdjFromVpnInterface(identifier, adj);
+                delAdjFromVpnInterface(identifier, adj, dpnId);
             }
         }
     }
@@ -739,68 +1141,113 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         return errorText.toString();
     }
 
-    private String getTunnelInterfaceFlowRef(BigInteger dpnId, short tableId, String ifName) {
-        return new StringBuilder().append(dpnId).append(tableId).append(ifName).toString();
+    private void addToLabelMapper(Long label, BigInteger dpnId, String prefix, List<String> nextHopIpList, Long vpnId,
+                                  String vpnInterfaceName, Long elanTag, boolean isSubnetRoute, String rd) {
+        Preconditions.checkNotNull(label, "label cannot be null or empty!");
+        Preconditions.checkNotNull(prefix, "prefix cannot be null or empty!");
+        Preconditions.checkNotNull(vpnId, "vpnId cannot be null or empty!");
+        Preconditions.checkNotNull(rd, "rd cannot be null or empty!");
+        if (!isSubnetRoute) {
+            // NextHop must be present for non-subnetroute entries
+            Preconditions.checkNotNull(nextHopIpList, "nextHopIp cannot be null or empty!");
+        }
+        LOG.info("Adding to label mapper : label {} dpn {} prefix {} nexthoplist {} vpnid {} vpnIntfcName {} rd {}", label, dpnId, prefix, nextHopIpList, vpnId, vpnInterfaceName, rd);
+        if (dpnId != null) {
+            InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
+                    .child(LabelRouteInfo.class, new LabelRouteInfoKey((long)label)).build();
+            LabelRouteInfoBuilder lriBuilder = new LabelRouteInfoBuilder();
+            lriBuilder.setLabel(label).setDpnId(dpnId).setPrefix(prefix).setNextHopIpList(nextHopIpList).setParentVpnid(vpnId)
+                    .setIsSubnetRoute(isSubnetRoute);
+            if (elanTag != null) {
+                lriBuilder.setElanTag(elanTag);
+            }
+            if (vpnInterfaceName != null) {
+                lriBuilder.setVpnInterfaceName(vpnInterfaceName);
+            }
+            lriBuilder.setParentVpnRd(rd);
+            VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(broker, rd);
+            if (vpnInstanceOpDataEntry != null) {
+                List<String> vpnInstanceNames = Arrays.asList(vpnInstanceOpDataEntry.getVpnInstanceName());
+                lriBuilder.setVpnInstanceList(vpnInstanceNames);
+            }
+            LabelRouteInfo lri = lriBuilder.build();
+            LOG.trace("Adding route info to label map: {}", lri);
+            VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, lriIid, lri);
+        } else {
+            LOG.trace("Can't add entry to label map for lable {},dpnId is null", label);
+        }
     }
 
+    public synchronized void addSubnetRouteFibEntryToDS(String rd, String vpnName, String prefix, String nextHop, int label,
+                                                        long elantag, BigInteger dpnId) {
 
+        SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
+        RouteOrigin origin = RouteOrigin.STATIC; // Only case when a route is considered as directly connected
+        VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
+                                                 .setLabel((long)label).setOrigin(origin.getValue())
+                                                 .addAugmentation(SubnetRoute.class, route).build();
 
-    public synchronized void addFibEntryToDS(String rd, String prefix,
-            String nexthop, int label) {
-
-        VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
-                    setNextHopAddressList(Arrays.asList(nexthop)).setLabel((long)label).build();
-        LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nexthop, label);
+        LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, elantag);
 
-        List<VrfEntry> vrfEntryList = new ArrayList<>();
-        vrfEntryList.add(vrfEntry);
+        //TODO: What should be parentVpnId? Get it from RD?
+        long vpnId = VpnUtil.getVpnId(broker, vpnName);
+        addToLabelMapper((long)label, dpnId, prefix, Arrays.asList(nextHop), vpnId, null, elantag, true, rd);
+        List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
 
         InstanceIdentifierBuilder<VrfTables> idBuilder =
-                    InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
+                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
 
         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
-                    setVrfEntry(vrfEntryList).build();
+                setVrfEntry(vrfEntryList).build();
 
         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
-    }
 
-    public synchronized void addSubnetRouteFibEntryToDS(String rd, String prefix,
-                                                        String nexthop, int label,long elantag) {
-
-        SubnetRoute route = new SubnetRouteBuilder().setElantag(elantag).build();
-
-        VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).
-                setNextHopAddressList(Arrays.asList(nexthop)).setLabel((long)label).addAugmentation(SubnetRoute
-                .class,route).build();
-        LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nexthop, label, elantag);
+        List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
+        if (vpnsToImportRoute.size() > 0) {
+            VrfEntry importingVrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
+                    .setLabel((long)label).setOrigin(RouteOrigin.SELF_IMPORTED.getValue())
+                    .addAugmentation(SubnetRoute.class, route).build();
+            List<VrfEntry> importingVrfEntryList = Arrays.asList(importingVrfEntry);
+            for (VpnInstance vpnInstance : vpnsToImportRoute) {
+                LOG.info("Exporting subnet route rd {} prefix {} nexthop {} label {} to vpn {}", rd, prefix, nextHop, label, vpnInstance.getVpnInstanceName());
+                String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
+                InstanceIdentifier<VrfTables> importingVrfTableId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(importingRd)).build();
+                VrfTables importingVrfTable = new VrfTablesBuilder().setRouteDistinguisher(importingRd).setVrfEntry(importingVrfEntryList).build();
+                VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, importingVrfTableId, importingVrfTable);
+            }
+        }
+    }
 
-        List<VrfEntry> vrfEntryList = new ArrayList<VrfEntry>();
-        vrfEntryList.add(vrfEntry);
+    public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
+                                                        SubnetRoute route) {
 
+        RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
+        VrfEntry vrfEntry = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
+                .setLabel((long)label).setOrigin(origin.getValue())
+                .addAugmentation(SubnetRoute.class, route).build();
+        LOG.debug("Created vrfEntry for {} nexthop {} label {} and elantag {}", prefix, nextHop, label, route.getElantag());
+        List<VrfEntry> vrfEntryList = Arrays.asList(vrfEntry);
         InstanceIdentifierBuilder<VrfTables> idBuilder =
                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
-
         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).
                 setVrfEntry(vrfEntryList).build();
-
         VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
     }
 
-    public synchronized void removeFibEntryFromDS(String rd, String prefix) {
-
-        LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
-
-        InstanceIdentifierBuilder<VrfEntry> idBuilder =
-            InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).child(VrfEntry.class, new VrfEntryKey(prefix));
-        InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
-        MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
-
+    public synchronized void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName){
+        VpnUtil.removeFibEntryFromDS(broker, rd, prefix, null /* nextHopToRemove */);
+        List<VpnInstance> vpnsToImportRoute = getVpnsImportingMyRoute(vpnName);
+        for (VpnInstance vpnInstance : vpnsToImportRoute) {
+            String importingRd = vpnInstance.getIpv4Family().getRouteDistinguisher();
+            LOG.info("Deleting imported subnet route rd {} prefix {} from vpn {}", rd, prefix, vpnInstance.getVpnInstanceName());
+            VpnUtil.removeFibEntryFromDS(broker, importingRd, prefix, null);
+        }
     }
 
     public synchronized void removeVrfFromDS(String rd) {
-        LOG.debug("Removing vrf table for  rd {}", rd);
+        LOG.debug("Removing vrf table for rd {}", rd);
 
         InstanceIdentifierBuilder<VrfTables> idBuilder =
                 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
@@ -810,7 +1257,7 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
     }
 
-    protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
+    protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
 
         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 
@@ -832,21 +1279,22 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                 adjacencies = new ArrayList<>();
             }
 
-            // FIXME 4: To be fixed with VPNManager patch
-            /*adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIp(adj.getNextHopIp())
-                    .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
 
+            adjacencies.add(new AdjacencyBuilder(adj).setLabel(label).setNextHopIpList(adj.getNextHopIpList())
+                    .setIpAddress(prefix).setKey(new AdjacencyKey(prefix)).build());
             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
-            VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug);
-
+            VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove());
             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
-            addExtraRoute(adj.getIpAddress(), adj.getNextHopIp(), rd, currVpnIntf.getVpnInstanceName(), (int) label, currVpnIntf.getName());*/
+            for (String nh : adj.getNextHopIpList()) {
+                addExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(), (int) label,
+                              currVpnIntf.getName());
+            }
 
         }
 
     }
 
-    protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj) {
+    protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterface> identifier, Adjacency adj, BigInteger dpnId) {
         Optional<VpnInterface> optVpnInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, identifier);
 
         if (optVpnInterface.isPresent()) {
@@ -864,19 +1312,19 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                     while (adjIt.hasNext()) {
                         Adjacency adjElem = adjIt.next();
                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
-                            // Commenting the release of ID here as it will be released by FIB
-                           /* VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
-                                    VpnUtil.getNextHopLabelKey(rd, adj.getIpAddress()));*/
                             adjIt.remove();
 
                             Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencies);
                             VpnInterface newVpnIntf = VpnUtil.getVpnInterface(currVpnIntf.getName(),
                                                                               currVpnIntf.getVpnInstanceName(),
-                                                                              aug);
+                                                                              aug, dpnId, currVpnIntf.isScheduledForRemove());
 
                             VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf);
 
-                            delExtraRoute(adj.getIpAddress(), rd, currVpnIntf.getVpnInstanceName());
+                            for (String nh : adj.getNextHopIpList()) {
+                                delExtraRoute(adj.getIpAddress(), nh, rd, currVpnIntf.getVpnInstanceName(),
+                                              currVpnIntf.getName());
+                            }
                             break;
                         }
 
@@ -887,38 +1335,75 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
     }
 
-    protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label, String intfName) {
+    protected void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label,
+                                 String intfName) {
 
         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
         VpnUtil.syncUpdate(
             broker,
             LogicalDatastoreType.OPERATIONAL,
-            VpnUtil.getVpnToExtrarouteIdentifier(
-                (rd != null) ? rd : routerID, destination),
-            VpnUtil.getVpnToExtraroute(destination, nextHop));
+            VpnUtil.getVpnToExtrarouteIdentifier( (rd != null) ? rd : routerID, destination),
+            VpnUtil.getVpnToExtraroute(destination, Arrays.asList(nextHop)));
 
+        BigInteger dpnId = null;
         if (intfName != null && !intfName.isEmpty()) {
-            BigInteger dpnId = InterfaceUtils.getDpnForInterface(interfaceManager, intfName);
+            dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
             if (nextHopIp == null || nextHopIp.isEmpty()) {
-                LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}", intfName, destination);
+                LOG.error("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
+                          intfName, destination);
+                return;
             }
             nextHop = nextHopIp;
         }
-        if (rd != null) {
-            addPrefixToBGP(rd, destination, nextHop, label);
+        List<String> nextHopIpList = Arrays.asList(nextHop);
+        addToLabelMapper((long)label, dpnId, destination, nextHopIpList, VpnUtil.getVpnId(broker, routerID),
+                intfName, null, false, rd);
+
+        // TODO (eperefr): This is a limitation to be stated in docs. When configuring static route to go to
+        // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
+        // first place.
+        InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(broker, nextHop);
+        if ( interVpnLink != null ) {
+            // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
+            // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
+            // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
+            // This is like leaking one of the Vpn2 routes towards Vpn1
+            boolean nexthopIsVpn2 = ( interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nextHop) );
+            String srcVpnUuid = (nexthopIsVpn2) ? interVpnLink.getSecondEndpoint().getVpnUuid().getValue()
+                                                : interVpnLink.getFirstEndpoint().getVpnUuid().getValue();
+            String dstVpnUuid = (nexthopIsVpn2) ? interVpnLink.getFirstEndpoint().getVpnUuid().getValue()
+                                                : interVpnLink.getSecondEndpoint().getVpnUuid().getValue();
+            String dstVpnRd = VpnUtil.getVpnRd(broker, dstVpnUuid);
+            long newLabel = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                                                VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
+            VpnUtil.leakRoute(broker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel);
         } else {
-            // ### add FIB route directly
-            addFibEntryToDS(routerID, destination, nextHop, label);
+            if (rd != null) {
+                addPrefixToBGP(rd, destination, nextHop, label);
+            } else {
+                // ### add FIB route directly
+                VpnUtil.addFibEntryToDS(broker, routerID, destination, nextHop, label, RouteOrigin.STATIC);
+            }
         }
     }
 
-    protected void delExtraRoute(String destination, String rd, String routerID) {
+    protected void delExtraRoute(String destination, String nextHop, String rd, String routerID, String intfName) {
+        if (intfName != null && !intfName.isEmpty()) {
+            BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
+            String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId);
+            if (nextHopIp == null || nextHopIp.isEmpty()) {
+                LOG.warn("NextHop for interface {} is null / empty. Failed advertising extra route for prefix {}",
+                        intfName, destination);
+            }
+            nextHop = nextHopIp;
+        }
+
         if (rd != null) {
-            removePrefixFromBGP(rd, destination);
+            removePrefixFromBGP(rd, destination, nextHop);
         } else {
             // ### add FIB route directly
-            removeFibEntryFromDS(routerID, destination);
+            VpnUtil.removeFibEntryFromDS(broker, routerID, destination, nextHop);
         }
     }
 
@@ -954,23 +1439,28 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                     // Vpn Interface removed => No more adjacencies from it.
                     // Hence clean up interface from vpn-dpn-interface list.
                     Adjacency adjacency = del.getAugmentation(Adjacencies.class).getAdjacency().get(0);
-                    Optional<Prefixes> prefixToInterface = Optional.absent();
-                    prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                    List<Prefixes> prefixToInterface = new ArrayList<>();
+                    Optional<Prefixes> prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
-                    if (!prefixToInterface.isPresent()) {
-                        // FIXME 5: To be fixed with VPNManager patch
-                        /*prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
-                                VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
-                                        VpnUtil.getIpPrefix(adjacency.getNextHopIp())));*/
+                    if (prefix.isPresent()) {
+                        prefixToInterface.add(prefix.get());
                     }
-                    if (prefixToInterface.isPresent()) {
+                    if (prefixToInterface.isEmpty()) {
+                        for (String nh : adjacency.getNextHopIpList()) {
+                            prefix = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                                     VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
+                                                                           VpnUtil.getIpPrefix(nh)));
+                            if (prefix.isPresent())
+                                prefixToInterface.add(prefix.get());
+                        }
+                    }
+                    for (Prefixes pref : prefixToInterface) {
                         VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
-                                VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
-                                        prefixToInterface.get().getIpAddress()),
-                                VpnUtil.DEFAULT_CALLBACK);
+                                       VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(), pref.getIpAddress()),
+                                       VpnUtil.DEFAULT_CALLBACK);
                         synchronized (interfaceName.intern()) {
-                            updateDpnDbs(prefixToInterface.get().getDpnId(), del.getVpnInstanceName(), interfaceName, false);
+                            updateDpnDbs(pref.getDpnId(), del.getVpnInstanceName(), interfaceName, false);
                         }
                     }
 //                    Long ifCnt = 0L;
@@ -1010,11 +1500,9 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
             //increment the vpn interface count in Vpn Instance Op Data
             //Long ifCnt = 0L;
             VpnInstanceOpDataEntry vpnInstOp = null;
-//            InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to
-//                    .vpn.id.VpnInstance>
+//            InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
 //                    updId = VpnUtil.getVpnInstanceToVpnIdIdentifier(update.getVpnInstanceName());
-//            Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id
-//                    .VpnInstance> updVpnInstance
+//            Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> updVpnInstance
 //                    = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, updId);
 //
 //            if (updVpnInstance.isPresent()) {
@@ -1027,19 +1515,16 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 //                    ifCnt = vpnInstOp.getVpnInterfaceCount();
 //                }
 //
-//                LOG.trace("VpnInterfaceOpListener update: interface name {} rd {} interface count in updated Vpn Op
-// Instance {}", interfaceName, rd, ifCnt);
+//                LOG.trace("VpnInterfaceOpListener update: interface name {} rd {} interface count in updated Vpn Op Instance {}", interfaceName, rd, ifCnt);
 //
 //                VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 //                        VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 //                        VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
 //            }
-
-            InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to
-                    .vpn.id.VpnInstance>
+//
+            InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
                     origId = VpnUtil.getVpnInstanceToVpnIdIdentifier(original.getVpnInstanceName());
-            Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id
-                    .VpnInstance> origVpnInstance
+            Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> origVpnInstance
                     = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, origId);
 
             if (origVpnInstance.isPresent()) {
@@ -1052,23 +1537,29 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 
                 if (vpnInstOp != null) {
                     Adjacency adjacency = original.getAugmentation(Adjacencies.class).getAdjacency().get(0);
-                    Optional<Prefixes> prefixToInterface = Optional.absent();
-                    prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                    List<Prefixes> prefixToInterfaceList = new ArrayList<>();
+                    Optional<Prefixes> prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
                             VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
                                     VpnUtil.getIpPrefix(adjacency.getIpAddress())));
-                    if (!prefixToInterface.isPresent()) {
-                        // FIXME 6: To be fixed with VPNManager patch
-                        /*prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
-                                VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
-                                        VpnUtil.getIpPrefix(adjacency.getNextHopIp())));*/
-                    }
                     if (prefixToInterface.isPresent()) {
+                        prefixToInterfaceList.add(prefixToInterface.get());
+                    } else {
+                        for (String adj : adjacency.getNextHopIpList()) {
+                            prefixToInterface = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                                          VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
+                                          VpnUtil.getIpPrefix(adj)));
+                            if (prefixToInterface.isPresent()) {
+                                prefixToInterfaceList.add(prefixToInterface.get());
+                            }
+                        }
+                    }
 //                        VpnUtil.delete(broker, LogicalDatastoreType.OPERATIONAL,
 //                                VpnUtil.getPrefixToInterfaceIdentifier(vpnInstOp.getVpnId(),
 //                                        prefixToInterface.get().getIpAddress()),
 //                                VpnUtil.DEFAULT_CALLBACK);
-                        synchronized (interfaceName.intern()) {
-                            updateDpnDbs(prefixToInterface.get().getDpnId(), original.getVpnInstanceName(), interfaceName, false);
+                    synchronized (interfaceName.intern()) {
+                        for (Prefixes prefix : prefixToInterfaceList) {
+                            updateDpnDbs(prefix.getDpnId(), original.getVpnInstanceName(), interfaceName, false);
                         }
                     }
                 }
@@ -1103,56 +1594,86 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
 //            if(vpnInstOp != null &&  vpnInstOp.getVpnInterfaceCount() != null) {
 //                ifCnt = vpnInstOp.getVpnInterfaceCount();
 //            }
-//
-//            LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}",
-//                    interfaceName, rd, ifCnt);
-//
+
+//            LOG.trace("VpnInterfaceOpListener add: interface name {} rd {} interface count in Vpn Op Instance {}", interfaceName, rd, ifCnt);
+
 //            VpnUtil.asyncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
 //                    VpnUtil.getVpnInstanceOpDataIdentifier(rd),
 //                    VpnUtil.updateIntfCntInVpnInstOpData(ifCnt + 1, rd), VpnUtil.DEFAULT_CALLBACK);
 
+
         }
     }
 
-    protected void updatePrefixesForDPN(BigInteger dpnId, UpdateRouteAction action) {
+    protected void handlePrefixesForDPNs(StateTunnelList stateTunnelList, UpdateRouteAction action) {
+        BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
+        BigInteger destDpnId;
+        String srcTepIp =  String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
+        String destTepIp = String.valueOf(stateTunnelList.getDstInfo().getTepIp().getValue());
 
-        LOG.info("Tunnel event triggered {} for Dpn:{} ", action.name(), dpnId);
         InstanceIdentifierBuilder<VpnInstances> idBuilder = InstanceIdentifier.builder(VpnInstances.class);
         InstanceIdentifier<VpnInstances> vpnInstancesId = idBuilder.build();
         Optional<VpnInstances> vpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vpnInstancesId);
+        long tunTypeVal = 0, vpnId;
+
+        if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeInternal.class) {
+            tunTypeVal = VpnConstants.ITMTunnelLocType.Internal.getValue();
+        } else if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeExternal.class) {
+            tunTypeVal = VpnConstants.ITMTunnelLocType.External.getValue();
+        } else if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeHwvtep.class){
+            tunTypeVal = VpnConstants.ITMTunnelLocType.Hwvtep.getValue();
+        } else {
+            tunTypeVal = VpnConstants.ITMTunnelLocType.Invalid.getValue();
+        }
+        LOG.trace("tunTypeVal is {}", tunTypeVal);
 
         if (vpnInstances.isPresent()) {
             List<VpnInstance> vpnInstanceList = vpnInstances.get().getVpnInstance();
             Iterator<VpnInstance> vpnInstIter = vpnInstanceList.iterator();
+            LOG.trace("vpnInstIter {}", vpnInstIter);
             while (vpnInstIter.hasNext()) {
                 VpnInstance vpnInstance = vpnInstIter.next();
+                LOG.trace("vpnInstance {}", vpnInstance);
+                vpnId = VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName());
                 try {
                     VpnAfConfig vpnConfig = vpnInstance.getIpv4Family();
+                    LOG.trace("vpnConfig {}", vpnConfig);
                     String rd = vpnConfig.getRouteDistinguisher();
                     if (rd == null || rd.isEmpty()) {
                         rd = vpnInstance.getVpnInstanceName();
+                        LOG.trace("rd is null or empty. Assigning VpnInstanceName to rd {}", rd);
                     }
-                    InstanceIdentifier<VpnToDpnList> id =
-                        VpnUtil.getVpnToDpnListIdentifier(rd, dpnId);
-                    Optional<VpnToDpnList> dpnInVpn =
-                        VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
-                    if (dpnInVpn.isPresent()) {
-                        // if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
-                        //    fibManager.populateFibOnNewDpn(dpnId, VpnUtil
-                        //        .getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
-                        // }
-                        List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                            .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
-                            vpnInterfaces = dpnInVpn.get().getVpnInterfaces();
-                        for (org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data
-                            .vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
+                    InstanceIdentifier<VpnToDpnList> srcId =
+                        VpnUtil.getVpnToDpnListIdentifier(rd, srcDpnId);
+                    Optional<VpnToDpnList> srcDpnInVpn =
+                            VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, srcId);
+                    if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
+                        destDpnId = new BigInteger(stateTunnelList.getDstInfo().getTepDeviceId());
+                        InstanceIdentifier<VpnToDpnList> destId =
+                                VpnUtil.getVpnToDpnListIdentifier(rd, destDpnId);
+                        Optional<VpnToDpnList> destDpnInVpn =
+                                VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, destId);
+                        if (!(srcDpnInVpn.isPresent() &&
+                                destDpnInVpn.isPresent())) {
+                            LOG.trace(" srcDpn {} - destDPN {}, do not share the VPN {} with rd {}.",
+                                    srcDpnId, destDpnId, vpnInstance.getVpnInstanceName(), rd);
+                            continue;
+                        }
+                    }
+                    if (srcDpnInVpn.isPresent()) {
+                        List<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                                .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
+                            vpnInterfaces = srcDpnInVpn.get().getVpnInterfaces();
+                        for (org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                                .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnInterface : vpnInterfaces) {
                             InstanceIdentifier<VpnInterface> vpnIntfId =
                                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getInterfaceName());
+                            LOG.trace("vpnInterface {}", vpnInterface);
                             InstanceIdentifier<Adjacencies> path =
                                 vpnIntfId.augmentation(Adjacencies.class);
                             Optional<Adjacencies> adjacencies =
                                 VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
-
+                            LOG.trace("adjacencies {}", adjacencies);
                             if (adjacencies.isPresent()) {
                                 List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
                                 Iterator<Adjacency> adjacencyIterator = adjacencyList.iterator();
@@ -1161,32 +1682,99 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                                     Adjacency adjacency = adjacencyIterator.next();
                                     try {
                                         if (action == UpdateRouteAction.ADVERTISE_ROUTE) {
-                                            //FIXME: TBD once odl-fib yang has nexthoplist and related changes follow
-                                            //bgpManager.addPrefix(rd, adjacency.getIpAddress(), adjacency.getNextHopIp(), adjacency.getLabel().intValue());
-
+                                            LOG.info("VPNInterfaceManager : Added Fib Entry rd {} prefix {} nextHop {} label {}",
+                                                     rd, adjacency.getIpAddress(), adjacency.getNextHopIpList(),
+                                                     adjacency.getLabel());
+//                                            vrf = new VrfEntryBuilder().set
+                                            if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
+                                                fibManager.handleRemoteRoute(true,
+                                                        new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId()),
+                                                        new BigInteger(stateTunnelList.getDstInfo().getTepDeviceId()),
+                                                        VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()),
+                                                        rd, adjacency.getIpAddress(), srcTepIp, destTepIp);
+                                            }
+                                            if (tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) {
+                                                bgpManager.advertisePrefix( rd, adjacency.getIpAddress(),
+                                                                            adjacency.getNextHopIpList(),
+                                                                            adjacency.getLabel().intValue());
+                                                fibManager.populateFibOnDpn(srcDpnId, vpnId, rd, srcTepIp, destTepIp);
+                                            }
+                                        }
+                                        else if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
+                                            LOG.info("VPNInterfaceManager : Removed Fib entry rd {} prefix {}",
+                                                     rd, adjacency.getIpAddress());
+                                            if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
+                                                fibManager.handleRemoteRoute(false, srcDpnId,
+                                                                new BigInteger(stateTunnelList.getDstInfo().getTepDeviceId()),
+                                                                vpnId, rd, adjacency.getIpAddress(), srcTepIp, destTepIp);
+                                            }
+                                            if (tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) {
+                                                bgpManager.withdrawPrefix(rd, adjacency.getIpAddress());
+                                                fibManager.cleanUpDpnForVpn(srcDpnId, vpnId, rd, srcTepIp, destTepIp);
+                                            }
                                         }
-                                        else if (action == UpdateRouteAction.WITHDRAW_ROUTE)
-                                            bgpManager.deletePrefix(rd, adjacency.getIpAddress());
                                     } catch (Exception e) {
                                         LOG.error("Exception when updating prefix {} in vrf {} to BGP",
                                             adjacency.getIpAddress(), rd);
                                     }
                                 }
+                            } else {
+                                LOG.trace("no adjacencies present for path {}.", path);
                             }
 
                         }
                         // if (action == UpdateRouteAction.WITHDRAW_ROUTE) {
                         //    fibManager.cleanUpDpnForVpn(dpnId, VpnUtil.getVpnId(broker, vpnInstance.getVpnInstanceName()), rd);
                         // }
+                    } else {
+                        LOG.trace("dpnInVpn check failed for srcDpnId {}.", srcDpnId);
                     }
                 } catch (Exception e) {
-                    LOG.error("updatePrefixesForDPN {} in vpn {} failed", dpnId, vpnInstance.getVpnInstanceName(), e);
+                    LOG.error("updatePrefixesForDPN {} in vpn {} failed", 0, vpnInstance.getVpnInstanceName(), e);
                 }
             }
+        } else {
+            LOG.trace("No vpn instances present.");
         }
     }
 
-       InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
+    void publishAddNotification(final BigInteger dpnId, final String vpnName, final String rd) {
+        LOG.debug("Sending notification for add dpn {} in vpn {} event ", dpnId, vpnName);
+        AddEventData data = new AddEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
+        AddDpnEvent event = new AddDpnEventBuilder().setAddEventData(data).build();
+        final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
+        Futures.addCallback(eventFuture, new FutureCallback<Object>() {
+            @Override
+            public void onFailure(Throwable error) {
+                LOG.warn("Error in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName, error);
+            }
+
+            @Override
+            public void onSuccess(Object arg) {
+                LOG.trace("Successful in notifying listeners for add dpn {} in vpn {} event ", dpnId, vpnName);
+            }
+        });
+    }
+
+    void publishRemoveNotification(final BigInteger dpnId, final String vpnName, final String rd) {
+        LOG.debug("Sending notification for remove dpn {} in vpn {} event ", dpnId, vpnName);
+        RemoveEventData data = new RemoveEventDataBuilder().setVpnName(vpnName).setRd(rd).setDpnId(dpnId).build();
+        RemoveDpnEvent event = new RemoveDpnEventBuilder().setRemoveEventData(data).build();
+        final ListenableFuture<? extends Object> eventFuture = notificationPublishService.offerNotification(event);
+        Futures.addCallback(eventFuture, new FutureCallback<Object>() {
+            @Override
+            public void onFailure(Throwable error) {
+                LOG.warn("Error in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName, error);
+            }
+
+            @Override
+            public void onSuccess(Object arg) {
+                LOG.trace("Successful in notifying listeners for remove dpn {} in vpn {} event ", dpnId, vpnName);
+            }
+        });
+    }
+
+    InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
         return InstanceIdentifier.builder(NeutronRouterDpns.class)
             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
@@ -1198,7 +1786,7 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
     }
 
     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName) {
-        BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, vpnInterfaceName);
+        BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
         if(dpId.equals(BigInteger.ZERO)) {
             LOG.warn("Could not retrieve dp id for interface {} to handle router {} association model", vpnInterfaceName, routerName);
             return;
@@ -1209,23 +1797,23 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
                 .OPERATIONAL, routerDpnListIdentifier);
         RouterInterfaces routerInterface = new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(vpnInterfaceName).build();
         if (optionalRouterDpnList.isPresent()) {
-            MDSALUtil.syncWrite(broker,  LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
+            MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
                     RouterInterfaces.class,  new RouterInterfacesKey(vpnInterfaceName)), routerInterface);
         } else {
-            MDSALUtil.syncUpdate(broker,  LogicalDatastoreType.OPERATIONAL,
+            MDSALUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL,
                     getRouterId(routerName),
                     new RouterDpnListBuilder().setRouterId(routerName).build());
             //VpnToDpnListBuilder vpnToDpnList = new VpnToDpnListBuilder().setDpnId(dpnId);
             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
             List<RouterInterfaces> routerInterfaces =  new ArrayList<>();
             routerInterfaces.add(routerInterface);
-            MDSALUtil.syncWrite(broker,  LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier,
+            MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier,
                     dpnVpnList.setRouterInterfaces(routerInterfaces).build());
         }
     }
 
     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName) {
-        BigInteger dpId = InterfaceUtils.getDpnForInterface(interfaceManager, vpnInterfaceName);
+        BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
         if(dpId.equals(BigInteger.ZERO)) {
             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
             return;
@@ -1248,8 +1836,8 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
             }
         }
     }
-       
-       protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,BigInteger dpId) {
+
+    protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,BigInteger dpId) {
         if(dpId.equals(BigInteger.ZERO)) {
             LOG.warn("Could not retrieve dp id for interface {} to handle router {} dissociation model", vpnInterfaceName, routerName);
             return;
@@ -1273,4 +1861,103 @@ public class VpnInterfaceManager extends AbstractDataChangeListener<VpnInterface
         }
     }
 
+    public void addMIPAdjacency(String vpnName,String vpnInterface, IpAddress prefix){
+
+        LOG.trace("Adding {} adjacency to VPN Interface {} ",prefix,vpnInterface);
+        InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
+        InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
+        Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
+        String nextHopIpAddr = null;
+        String nextHopMacAddress = null;
+        String ip = prefix.getIpv4Address().getValue();
+        if (adjacencies.isPresent()) {
+            List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
+            ip = VpnUtil.getIpPrefix(ip);
+            for (Adjacency adjacs : adjacencyList) {
+                if (adjacs.getMacAddress() != null && !adjacs.getMacAddress().isEmpty()) {
+                    nextHopIpAddr = adjacs.getIpAddress();
+                    nextHopMacAddress = adjacs.getMacAddress();
+                    break;
+                }
+            }
+            if (nextHopMacAddress != null && ip != null) {
+                synchronized (vpnInterface.intern()) {
+                    String rd = VpnUtil.getVpnRd(broker, vpnName);
+                    long label =
+                            VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                                    VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ip));
+                    String nextHopIp = nextHopIpAddr.split("/")[0];
+                    Adjacency newAdj = new AdjacencyBuilder().setIpAddress(ip).setKey
+                            (new AdjacencyKey(ip)).setNextHopIpList(Arrays.asList(nextHopIp)).build();
+                    adjacencyList.add(newAdj);
+                    Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
+                    VpnInterface newVpnIntf = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(vpnInterface)).
+                            setName(vpnInterface).setVpnInstanceName(vpnName).addAugmentation(Adjacencies.class, aug).build();
+                    VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
+                }
+                LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
+            }
+        }
+
+    }
+
+    public void removeMIPAdjacency(String vpnName, String vpnInterface, IpAddress prefix) {
+        String ip = VpnUtil.getIpPrefix(prefix.getIpv4Address().getValue());
+        LOG.trace("Removing {} adjacency from Old VPN Interface {} ",ip,vpnInterface);
+        InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
+        InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
+        Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, path);
+        if (adjacencies.isPresent()) {
+            synchronized (vpnInterface.intern()) {
+                InstanceIdentifier<Adjacency> adjacencyIdentifier = InstanceIdentifier.builder(VpnInterfaces.class).
+                        child(VpnInterface.class, new VpnInterfaceKey(vpnInterface)).augmentation(Adjacencies.class)
+                        .child(Adjacency.class, new AdjacencyKey(ip)).build();
+                MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, adjacencyIdentifier);
+            }
+            LOG.trace("Successfully Deleted Adjacency into VpnInterface {}", vpnInterface);
+        }
+    }
+
+    class TunnelInterfaceStateListener extends AbstractDataChangeListener<StateTunnelList>  {
+        public TunnelInterfaceStateListener(final DataBroker db, VpnInterfaceManager vpnIfMgr) {
+            super(StateTunnelList.class);
+        }
+
+
+        @Override
+        protected void remove(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList del) {
+            LOG.trace("Tunnel deletion---- {}", del);
+            handlePrefixesForDPNs(del, UpdateRouteAction.WITHDRAW_ROUTE);
+        }
+
+        @Override
+        protected void update(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList original, StateTunnelList update) {
+            LOG.trace("Tunnel updation---- {}", update);
+            LOG.trace("ITM Tunnel {} of type {} state event changed from :{} to :{}",
+                    update.getTunnelInterfaceName(),
+                    fibManager.getTransportTypeStr(update.getTransportType().toString()),
+                    original.isTunnelState(), update.isTunnelState());
+                //withdraw all prefixes in all vpns for this dpn
+            boolean isTunnelUp = update.isTunnelState();
+                handlePrefixesForDPNs(update,isTunnelUp ?  UpdateRouteAction.ADVERTISE_ROUTE :
+                                                            UpdateRouteAction.WITHDRAW_ROUTE);
+        }
+
+        @Override
+        protected void add(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList add) {
+            LOG.trace("Tunnel addition---- {}", add);
+
+            if(!add.isTunnelState()) {
+                LOG.trace(  "Tunnel {} is not yet UP.",
+                            add.getTunnelInterfaceName());
+                return;
+            } else {
+                LOG.trace("ITM Tunnel ,type {} ,State is UP b/w src: {} and dest: {}",
+                        fibManager.getTransportTypeStr(add.getTransportType().toString()),
+                        add.getSrcInfo().getTepDeviceId(), add.getDstInfo().getTepDeviceId());
+                handlePrefixesForDPNs(add, UpdateRouteAction.ADVERTISE_ROUTE);
+            }
+        }
+    }
+
 }
index 1fde4894af95be87d7edc58b1e9d725977e5f4e5..381095172cdfa739cb35753eef927c4c72cabe72 100644 (file)
@@ -22,6 +22,7 @@ import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
@@ -55,17 +56,14 @@ import com.google.common.util.concurrent.Futures;
 
 public class VpnManager extends AbstractDataChangeListener<VpnInstance> implements AutoCloseable {
     private static final Logger LOG = LoggerFactory.getLogger(VpnManager.class);
-    private ListenerRegistration<DataChangeListener> listenerRegistration, fibListenerRegistration, opListenerRegistration;
-    private ConcurrentMap<String, Runnable> vpnOpMap = new ConcurrentHashMap<>();
-    private static final ThreadFactory threadFactory = new ThreadFactoryBuilder()
-        .setNameFormat("NV-VpnMgr-%d").build();
-    private ExecutorService executorService = Executors.newSingleThreadExecutor(threadFactory);
+    private ListenerRegistration<DataChangeListener> listenerRegistration, fibListenerRegistration;
+    private ConcurrentMap<String, Runnable> vpnOpMap = new ConcurrentHashMap<String, Runnable>();
+    private ExecutorService executorService = Executors.newSingleThreadExecutor();
     private final DataBroker broker;
     private final IBgpManager bgpManager;
     private IdManagerService idManager;
     private VpnInterfaceManager vpnInterfaceManager;
     private final FibEntriesListener fibListener;
-    private final VpnInstanceOpListener vpnInstOpListener;
     private NotificationService notificationService;
 
     private static final FutureCallback<Void> DEFAULT_CALLBACK =
@@ -78,21 +76,21 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
                 @Override
                 public void onFailure(Throwable error) {
                     LOG.error("Error in Datastore operation", error);
-                }
+                };
             };
 
     /**
      * Listens for data change related to VPN Instance
      * Informs the BGP about VRF information
      *
-     * @param db - dataBroker reference
+     * @param db dataBroker reference
+     * @param bgpManager Used to advertise routes to the BGP Router
      */
     public VpnManager(final DataBroker db, final IBgpManager bgpManager) {
         super(VpnInstance.class);
         broker = db;
         this.bgpManager = bgpManager;
         this.fibListener = new FibEntriesListener();
-        this.vpnInstOpListener = new VpnInstanceOpListener();
         registerListener(db);
     }
 
@@ -102,9 +100,6 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
                     getWildCardPath(), VpnManager.this, DataChangeScope.SUBTREE);
             fibListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
                     getFibEntryListenerPath(), fibListener, DataChangeScope.BASE);
-            opListenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
-                    getVpnInstanceOpListenerPath(), vpnInstOpListener, DataChangeScope.SUBTREE);
-
         } catch (final Exception e) {
             LOG.error("VPN Service DataChange listener registration fail !", e);
             throw new IllegalStateException("VPN Service registration Listener failed.", e);
@@ -125,10 +120,11 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
         VpnInstanceOpDataEntry vpnOpEntry = null;
         Long intfCount = 0L;
         Long currentIntfCount = 0L;
+        Integer retryCount = 1;
         long timeout = VpnConstants.MIN_WAIT_TIME_IN_MILLISECONDS;
         Optional<VpnInstanceOpDataEntry> vpnOpValue = null;
         vpnOpValue = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
-                VpnUtil.getVpnInstanceOpDataIdentifier(rd));
+                    VpnUtil.getVpnInstanceOpDataIdentifier(rd));
 
         if ((vpnOpValue != null) && (vpnOpValue.isPresent())) {
             vpnOpEntry = vpnOpValue.get();
@@ -195,12 +191,26 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
                         // There is no progress by FibManager in removing all the interfaces even after good time!
                         // In either case, let us quit and take our chances.
                         //TODO(vpnteam): L3VPN refactoring to take care of this case.
-                        LOG.info("VPNInstance bailing out of wait loop as currentIntfCount is {} for for rd {}, vpnname {}",
-                                currentIntfCount, rd, vpnName);
-                        break;
+                        if (retryCount > 0) {
+                            retryCount--;
+                            LOG.info("Retrying clearing vpn with vpnname {} rd {} since currentIntfCount {} ", vpnName, rd, currentIntfCount);
+                            if (currentIntfCount > 0L){
+                                LOG.info("Current interface count for vpn {} and rd {} is not zero and so retrying ...", vpnName, rd);
+                                intfCount = currentIntfCount;
+                            } else {
+                                LOG.info("Current interface count is zero but instance Op for vpn {} and rd {} not cleared yet. Waiting for 5 more seconds.", vpnName, rd);
+                                intfCount = 1L;
+                            }
+                        } else {
+                            LOG.info("VPNInstance bailing out of wait loop as currentIntfCount is {} and max retries exceeded for for rd {}, vpnname {}",
+                                    currentIntfCount, rd, vpnName);
+                            break;
+                        }
                     } else {
                         // There is some progress by FibManager, so let us give it some more time!
                         intfCount = currentIntfCount;
+                        retryCount = 1;
+                        LOG.info("current interface count {} for vpn {} and rd {} showing progress, waiting for it to drive to 0.", currentIntfCount, vpnName, rd);
                     }
                 } else {
                     // There is no VPNOPEntry.  Something else happened on the system !
@@ -213,6 +223,7 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
         LOG.info("Returned out of waiting for  Op Data removal for rd {}, vpnname {}", rd, vpnName);
     }
 
+
     @Override
     protected void remove(InstanceIdentifier<VpnInstance> identifier, VpnInstance del) {
         LOG.trace("Remove VPN event key: {}, value: {}", identifier, del);
@@ -235,14 +246,18 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
         } else {
             waitForOpRemoval(vpnName, vpnName);
         }
+
         // Clean up VpnInstanceToVpnId from Config DS
+        VpnUtil.removeVpnIdToVpnInstance(broker, vpnId);
         VpnUtil.removeVpnInstanceToVpnId(broker, vpnName);
         LOG.trace("Removed vpnIdentifier for  rd{} vpnname {}", rd, vpnName);
         if (rd != null) {
-            try {
-                bgpManager.deleteVrf(rd);
-            } catch (Exception e) {
-                LOG.error("Exception when removing VRF from BGP for RD {} in VPN {} exception " + e, rd, vpnName);
+            synchronized (rd.intern()) {
+                try {
+                    bgpManager.deleteVrf(rd);
+                } catch (Exception e) {
+                    LOG.error("Exception when removing VRF from BGP for RD {} in VPN {} exception " + e, rd, vpnName);
+                }
             }
 
             // Clean up VPNExtraRoutes Operational DS
@@ -252,7 +267,9 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
             VpnUtil.removeVpnOpInstance(broker, rd);
         } else {
             // Clean up FIB Entries Config DS
-            VpnUtil.removeVrfTableForVpn(broker, vpnName);
+            synchronized (vpnName.intern()) {
+                VpnUtil.removeVrfTableForVpn(broker, vpnName);
+            }
 
             // Clean up VPNExtraRoutes Operational DS
             VpnUtil.removeVpnExtraRouteForVpn(broker, vpnName);
@@ -268,7 +285,6 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
         VpnUtil.removeL3nexthopForVpnId(broker, vpnId);
 
         // Release the ID used for this VPN back to IdManager
-
         VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME, vpnName);
     }
 
@@ -279,61 +295,67 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
     }
 
     @Override
-    protected void add(InstanceIdentifier<VpnInstance> identifier,
-            VpnInstance value) {
+    protected void add(InstanceIdentifier<VpnInstance> identifier, VpnInstance value) {
         LOG.trace("Add VPN event key: {}, value: {}", identifier, value);
         VpnAfConfig config = value.getIpv4Family();
         String rd = config.getRouteDistinguisher();
+        String vpnInstanceName = value.getVpnInstanceName();
 
-        long vpnId = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, value.getVpnInstanceName());
+        long vpnId = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, vpnInstanceName);
         LOG.trace("VPN instance to ID generated.");
         org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance
-            vpnInstanceToVpnId = VpnUtil.getVpnInstanceToVpnId(value.getVpnInstanceName(), vpnId,
-                                                                    (rd != null) ? rd : value.getVpnInstanceName());
+            vpnInstanceToVpnId = VpnUtil.getVpnInstanceToVpnId(vpnInstanceName, vpnId, (rd != null) ? rd
+                                                                                                    : vpnInstanceName);
+
+        syncWrite(LogicalDatastoreType.CONFIGURATION, VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnInstanceName),
+                  vpnInstanceToVpnId, DEFAULT_CALLBACK);
+
+        VpnIds
+                vpnIdToVpnInstance = VpnUtil.getVpnIdToVpnInstance(vpnId, value.getVpnInstanceName(),
+                (rd != null) ? rd : value.getVpnInstanceName(), (rd != null)/*isExternalVpn*/) ;
 
         syncWrite(LogicalDatastoreType.CONFIGURATION,
-                   VpnUtil.getVpnInstanceToVpnIdIdentifier(value.getVpnInstanceName()),
-                   vpnInstanceToVpnId, DEFAULT_CALLBACK);
+                VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId),
+                vpnIdToVpnInstance, DEFAULT_CALLBACK);
 
         IFibManager fibManager = vpnInterfaceManager.getFibManager();
         try {
-            String cachedTransType = fibManager.getReqTransType();
+            String cachedTransType = fibManager.getConfTransType();
             LOG.trace("Value for confTransportType is " + cachedTransType);
             if (cachedTransType.equals("Invalid")) {
                 try {
                     fibManager.setConfTransType("L3VPN", "VXLAN");
-                    LOG.trace("setting it to vxlan now");
                 } catch (Exception e) {
+                    LOG.trace("Exception caught setting the cached value for transportType");
                     LOG.error(e.getMessage());
                 }
             } else {
                 LOG.trace(":cached val is neither unset/invalid. NO-op.");
             }
         } catch (Exception e) {
-            System.out.println("Exception caught accessing the cached value for transportType");
             LOG.error(e.getMessage());
         }
 
         if(rd == null) {
-            VpnInstanceOpDataEntryBuilder builder = new VpnInstanceOpDataEntryBuilder();
-            builder.setVrfId(value.getVpnInstanceName()).setVpnId(vpnId);
-            builder.setVpnInterfaceCount(0L);
-            syncWrite(LogicalDatastoreType.OPERATIONAL,
-                    VpnUtil.getVpnInstanceOpDataIdentifier(value.getVpnInstanceName()),
-                    builder.build(), DEFAULT_CALLBACK);
+            VpnInstanceOpDataEntryBuilder builder =
+                new VpnInstanceOpDataEntryBuilder().setVrfId(vpnInstanceName).setVpnId(vpnId)
+                                                   .setVpnInstanceName(vpnInstanceName)
+                                                   .setVpnInterfaceCount(0L);
+            syncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil.getVpnInstanceOpDataIdentifier(vpnInstanceName),
+                      builder.build(), DEFAULT_CALLBACK);
 
         } else {
-            VpnInstanceOpDataEntryBuilder builder = new VpnInstanceOpDataEntryBuilder();
-            builder.setVrfId(rd).setVpnId(vpnId);
-            builder.setVpnInterfaceCount(0L);
+            VpnInstanceOpDataEntryBuilder builder =
+                new VpnInstanceOpDataEntryBuilder().setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnInstanceName)
+                                                   .setVpnInterfaceCount(0L);
             syncWrite(LogicalDatastoreType.OPERATIONAL,
-                       VpnUtil.getVpnInstanceOpDataIdentifier(rd),
-                       builder.build(), DEFAULT_CALLBACK);
+                      VpnUtil.getVpnInstanceOpDataIdentifier(rd),
+                      builder.build(), DEFAULT_CALLBACK);
 
             List<VpnTarget> vpnTargetList = config.getVpnTargets().getVpnTarget();
 
-            List<String> ertList = new ArrayList<>();
-            List<String> irtList = new ArrayList<>();
+            List<String> ertList = new ArrayList<String>();
+            List<String> irtList = new ArrayList<String>();
 
             for (VpnTarget vpnTarget : vpnTargetList) {
                 if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.ExportExtcommunity) {
@@ -361,14 +383,15 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
         if(optionalVpnInterfaces.isPresent()) {
             List<VpnInterface> vpnInterfaces = optionalVpnInterfaces.get().getVpnInterface();
             for(VpnInterface vpnInterface : vpnInterfaces) {
-                if(vpnInterface.getVpnInstanceName().equals(value.getVpnInstanceName())) {
-                    LOG.debug("VpnInterface {} will be added from VPN {}", vpnInterface.getName(), value.getVpnInstanceName());
+                if(vpnInterface.getVpnInstanceName().equals(vpnInstanceName)) {
+                    LOG.debug("VpnInterface {} will be added from VPN {}", vpnInterface.getName(), vpnInstanceName);
                     vpnInterfaceManager.add(
                                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()), vpnInterface);
 
                 }
             }
         }
+        vpnInterfaceManager.handleVpnsExportingRoutes(vpnInstanceName, rd);
     }
 
     public boolean isVPNConfigured() {
@@ -418,15 +441,6 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
             }
             fibListenerRegistration = null;
         }
-        if (opListenerRegistration != null) {
-            try {
-                opListenerRegistration.close();
-            } catch (final Exception e) {
-                LOG.error("Error when cleaning up VPN Instance Operational entries DataChangeListener.", e);
-            }
-            opListenerRegistration = null;
-        }
-
         LOG.trace("VPN Manager Closed");
     }
 
@@ -536,46 +550,4 @@ public class VpnManager extends AbstractDataChangeListener<VpnInstance> implemen
             }
         }
     }
-
-    class VpnInstanceOpListener extends AbstractDataChangeListener<VpnInstanceOpDataEntry> {
-
-        public VpnInstanceOpListener() {
-            super(VpnInstanceOpDataEntry.class);
-        }
-
-        @Override
-        protected void remove(InstanceIdentifier<VpnInstanceOpDataEntry> identifier, VpnInstanceOpDataEntry del) {
-
-        }
-
-        @Override
-        protected void update(InstanceIdentifier<VpnInstanceOpDataEntry> identifier, VpnInstanceOpDataEntry original, VpnInstanceOpDataEntry update) {
-            final VpnInstanceOpDataEntryKey key = identifier.firstKeyOf(VpnInstanceOpDataEntry.class, VpnInstanceOpDataEntryKey.class);
-            String vpnName = key.getVrfId();
-
-//            LOG.trace("VpnInstanceOpListener update: vpn name {} interface count in Old VpnOp Instance {} in New " +
-//                    "VpnOp Instance {}" ,
-//                            vpnName, original.getVpnInterfaceCount(), update.getVpnInterfaceCount() );
-//
-//            //if((original.getVpnToDpnList().size() != update.getVpnToDpnList().size()) && (update.getVpnToDpnList()
-// .size() == 0)) {
-//            if((original.getVpnInterfaceCount() != update.getVpnInterfaceCount()) && (update.getVpnInterfaceCount()
-//                    == 0)) {
-//                notifyTaskIfRequired(vpnName);
-//           }
-       }
-
-//        private void notifyTaskIfRequired(String vpnName) {
-//            Runnable notifyTask = vpnOpMap.remove(vpnName);
-//            if (notifyTask == null) {
-//                LOG.trace("VpnInstanceOpListener update: No Notify Task queued for vpnName {}", vpnName);
-//                return;
-//            }
-//            executorService.execute(notifyTask);
-//        }
-
-        @Override
-        protected void add(InstanceIdentifier<VpnInstanceOpDataEntry> identifier, VpnInstanceOpDataEntry add) {
-        }
-    }
 }
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnRpcServiceImpl.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnRpcServiceImpl.java
new file mode 100644 (file)
index 0000000..303e913
--- /dev/null
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.*;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
+import org.opendaylight.yangtools.yang.common.RpcError;
+import org.opendaylight.yangtools.yang.common.RpcError.ErrorType;
+import org.opendaylight.yangtools.yang.common.RpcResult;
+import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Future;
+
+public class VpnRpcServiceImpl implements VpnRpcService {
+
+    private static final Logger LOG = LoggerFactory.getLogger(VpnRpcServiceImpl.class);
+    private IdManagerService idManager;
+    private VpnInterfaceManager vpnInterfaceMgr;
+    private DataBroker dataBroker;
+
+    public VpnRpcServiceImpl(IdManagerService idManager, VpnInterfaceManager vpnIfaceMgr, DataBroker dataBroker) {
+        this.idManager = idManager;
+        this.vpnInterfaceMgr = vpnIfaceMgr;
+        this.dataBroker = dataBroker;
+    }
+
+    /**
+     * to generate label for the given ip prefix from the associated VPN
+     *
+     */
+    @Override
+    public Future<RpcResult<GenerateVpnLabelOutput>> generateVpnLabel(GenerateVpnLabelInput input) {
+        String vpnName = input.getVpnName();
+        String ipPrefix = input.getIpPrefix();
+        SettableFuture<RpcResult<GenerateVpnLabelOutput>> futureResult = SettableFuture.create();
+        String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
+        long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                        VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ipPrefix));
+        if (label == 0) {
+            String msg = String.format("Could not retrieve the label for prefix {} in VPN {}", ipPrefix, vpnName);
+            LOG.error(msg);
+            futureResult.set(RpcResultBuilder.<GenerateVpnLabelOutput>failed().withError(ErrorType.APPLICATION, msg)
+                                                                              .build());
+        } else {
+            GenerateVpnLabelOutput output = new GenerateVpnLabelOutputBuilder().setLabel(label).build();
+            futureResult.set(RpcResultBuilder.success(output).build());
+        }
+        return futureResult;
+    }
+
+    /**
+     * to remove label for the given ip prefix from the associated VPN
+     *
+     */
+    @Override
+    public Future<RpcResult<Void>> removeVpnLabel(RemoveVpnLabelInput input) {
+        String vpnName = input.getVpnName();
+        String ipPrefix = input.getIpPrefix();
+        String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
+        SettableFuture<RpcResult<Void>> futureResult = SettableFuture.create();
+        VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                VpnUtil.getNextHopLabelKey((rd != null) ? rd : vpnName, ipPrefix));
+        futureResult.set(RpcResultBuilder.<Void>success().build());
+        return futureResult;
+    }
+
+    private Collection<RpcError> validateAddStaticRouteInput(AddStaticRouteInput input) {
+        Collection<RpcError> rpcErrors = new ArrayList<RpcError>();
+        String destination = input.getDestination();
+        String vpnInstanceName = input.getVpnInstanceName();
+        String nexthop = input.getNexthop();
+        if ( destination == null || destination.isEmpty() ) {
+            String message = "destination parameter is mandatory";
+            rpcErrors.add(RpcResultBuilder.newError(RpcError.ErrorType.PROTOCOL, "addStaticRoute", message));
+        }
+        if ( vpnInstanceName == null || vpnInstanceName.isEmpty() ) {
+            String message = "vpnInstanceName parameter is mandatory";
+            rpcErrors.add(RpcResultBuilder.newError(RpcError.ErrorType.PROTOCOL, "addStaticRoute", message));
+        }
+        if ( nexthop == null || nexthop.isEmpty() ) {
+            String message = "nexthop parameter is mandatory";
+            rpcErrors.add(RpcResultBuilder.newError(RpcError.ErrorType.PROTOCOL, "addStaticRoute", message));
+        }
+        return rpcErrors;
+    }
+
+    @Override
+    public Future<RpcResult<AddStaticRouteOutput>> addStaticRoute(AddStaticRouteInput input) {
+
+        SettableFuture<RpcResult<AddStaticRouteOutput>> result = SettableFuture.create();
+        String destination = input.getDestination();
+        String vpnInstanceName = input.getVpnInstanceName();
+        String nexthop = input.getNexthop();
+        Long label = input.getLabel();
+        LOG.info("Adding static route for Vpn {} with destination {}, nexthop {} and label {}",
+                 vpnInstanceName, destination, nexthop, label);
+
+        Collection<RpcError> rpcErrors = validateAddStaticRouteInput(input);
+        if ( !rpcErrors.isEmpty() ) {
+            result.set(RpcResultBuilder.<AddStaticRouteOutput>failed().withRpcErrors(rpcErrors).build());
+            return result;
+        }
+
+        if ( label == null || label == 0 ) {
+            label = (long) VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                                               VpnUtil.getNextHopLabelKey(vpnInstanceName, destination));
+            if ( label == 0 ) {
+                String message = "Unable to retrieve a new Label for the new Route";
+                result.set(RpcResultBuilder.<AddStaticRouteOutput>failed().withError(RpcError.ErrorType.APPLICATION,
+                                                                                     message).build());
+                return result;
+            }
+        }
+
+        String vpnRd = VpnUtil.getVpnRd(dataBroker, input.getVpnInstanceName());
+        if ( vpnRd == null ) {
+            String message = "Could not find Route-Distinguisher for VpnName " + vpnInstanceName;
+            result.set(RpcResultBuilder.<AddStaticRouteOutput>failed().withError(RpcError.ErrorType.APPLICATION,
+                                                                                 message).build());
+            return result;
+        }
+
+        InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(dataBroker, nexthop);
+        if ( interVpnLink != null ) {
+            // A static route pointing to an InterVpnLink endpoint: just write the VrfEntry
+            VpnUtil.addFibEntryToDS(dataBroker, vpnRd, destination, nexthop, label.intValue(), RouteOrigin.STATIC);
+        } else {
+            vpnInterfaceMgr.addExtraRoute(destination, nexthop, vpnRd, null /*routerId */, label.intValue(),
+                                          null /* intfName */);
+        }
+
+        AddStaticRouteOutput labelOutput = new AddStaticRouteOutputBuilder().setLabel(label).build();
+        result.set(RpcResultBuilder.success(labelOutput).build());
+        return result;
+    }
+
+    private Collection<RpcError> validateRemoveStaticRouteInput(RemoveStaticRouteInput input) {
+        Collection<RpcError> rpcErrors = new ArrayList<RpcError>();
+        String destination = input.getDestination();
+        String vpnInstanceName = input.getVpnInstanceName();
+        String nexthop = input.getNexthop();
+        if ( destination == null || destination.isEmpty() ) {
+            String message = "destination parameter is mandatory";
+            rpcErrors.add(RpcResultBuilder.newError(RpcError.ErrorType.PROTOCOL, "removeStaticRoute", message));
+        }
+        if ( vpnInstanceName == null || vpnInstanceName.isEmpty() ) {
+            String message = "vpnInstanceName parameter is mandatory";
+            rpcErrors.add(RpcResultBuilder.newError(RpcError.ErrorType.PROTOCOL, "removeStaticRoute", message));
+        }
+        if ( nexthop == null || nexthop.isEmpty() ) {
+            String message = "nexthop parameter is mandatory";
+            rpcErrors.add(RpcResultBuilder.newError(RpcError.ErrorType.PROTOCOL, "removeStaticRoute", message));
+        }
+        return rpcErrors;
+    }
+
+    @Override
+    public Future<RpcResult<Void>> removeStaticRoute(RemoveStaticRouteInput input) {
+
+        SettableFuture<RpcResult<Void>> result = SettableFuture.create();
+
+        String destination = input.getDestination();
+        String vpnInstanceName = input.getVpnInstanceName();
+        String nexthop = input.getNexthop();
+        LOG.info("Removing static route with destination={}, nexthop={} in VPN={}",
+                 destination, nexthop, vpnInstanceName);
+        Collection<RpcError> rpcErrors = validateRemoveStaticRouteInput(input);
+        if ( !rpcErrors.isEmpty() ) {
+            result.set(RpcResultBuilder.<Void>failed().withRpcErrors(rpcErrors).build());
+            return result;
+        }
+
+        String vpnRd = VpnUtil.getVpnRd(dataBroker, input.getVpnInstanceName());
+        if ( vpnRd == null ) {
+            String message = "Could not find Route-Distinguisher for VpnName " + vpnInstanceName;
+            result.set(RpcResultBuilder.<Void>failed().withError(RpcError.ErrorType.APPLICATION, message).build());
+            return result;
+        }
+
+        InterVpnLink interVpnLink = VpnUtil.getInterVpnLinkByEndpointIp(dataBroker, nexthop);
+        if ( interVpnLink != null ) {
+            // A static route pointing to an InterVpnLink endpoint: just remove the VrfEntry from DS
+            VpnUtil.removeFibEntryFromDS(dataBroker,  vpnRd, destination, nexthop);
+        } else {
+            vpnInterfaceMgr.delExtraRoute(destination, nexthop, vpnRd, null /*routerId*/, null /*intfName*/);
+        }
+        result.set(RpcResultBuilder.<Void>success().build());
+
+        return result;
+    }
+
+}
index e21f529fcea8bb111c514d94a5c11600fbf4dd1e..33fa34ce29405df3557523714f9ce06818571d2f 100644 (file)
@@ -7,10 +7,7 @@
  */
 package org.opendaylight.netvirt.vpnmanager;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
+import java.util.*;
 
 import com.google.common.base.Preconditions;
 
@@ -34,9 +31,9 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev15060
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.subnet.to.dpn.VpnInterfaces;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 
 import java.math.BigInteger;
@@ -118,7 +115,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                     return;
                 }
                 logger.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
-                Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<>();
+                Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<BigInteger, SubnetToDpn>();
                 SubnetOpDataEntry subOpEntry = null;
                 BigInteger dpnId = null;
                 BigInteger nhDpnId = null;
@@ -134,7 +131,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 }
                 subOpBuilder.setVrfId(rd);
                 subOpBuilder.setVpnName(vpnName);
-                subOpBuilder.setSubnetToDpn(new ArrayList<>());
+                subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>());
                 subOpBuilder.setRouteAdvState(TaskState.Na);
                 subOpBuilder.setElanTag(elanTag);
 
@@ -167,11 +164,12 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                         }
                     }
                     if (subDpnMap.size() > 0) {
-                        subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
+                        subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>(subDpnMap.values()));
                     }
                 }
 
                 if (nhDpnId != null) {
+                    logger.info("Next-Hop dpn {} is available for rd {} subnetIp {} vpn {}", nhDpnId, rd, subnetIp, vpnName);
                     subOpBuilder.setNhDpnId(nhDpnId);
                     try {
                         /*
@@ -187,19 +185,8 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
                         subOpBuilder.setRouteAdvState(TaskState.Pending);
                     }
-                } else {
-                    try {
-                        /*
-                        Write the subnet route entry to the FIB.
-                        NOTE: Will not advertise to BGP as NextHopDPN is not available yet.
-                        */
-                        int label = getLabel(rd, subnetIp);
-                        addSubnetRouteToFib(rd, subnetIp, null, vpnName, elanTag, label);
-                    } catch (Exception ex) {
-                        logger.error("onSubnetAddedToVpn: FIB rules writing for subnet {} with exception {} " +
-                                subnetId.getValue(), ex);
-                        subOpBuilder.setRouteAdvState(TaskState.Pending);
-                    }
+                }else{
+                    logger.info("Next-Hop dpn is unavailable for rd {} subnetIp {} vpn {}", rd, subnetIp, vpnName);
                 }
 
                 subOpEntry = subOpBuilder.build();
@@ -220,12 +207,12 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         if (!notification.isExternalVpn()) {
             return;
         }
-        logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed to vpn");
+        logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed from vpn");
         //TODO(vivek): Change this to use more granularized lock at subnetId level
         synchronized (this) {
             try {
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
-                        child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
+                    child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
                         LogicalDatastoreType.OPERATIONAL,
@@ -235,53 +222,40 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                             " not available in datastore");
                     return;
                 }
-                SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
-                List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
-                for (SubnetToDpn subDpn: subDpnList) {
-                    List<VpnInterfaces> vpnIntfList = subDpn.getVpnInterfaces();
-                    for (VpnInterfaces vpnIntf: vpnIntfList) {
-                        subOpDpnManager.removePortOpDataEntry(vpnIntf.getInterfaceName());
-                    }
-                }
-                //Removing Stale Ports in portOpData
+
+                /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
+                 * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
+                 * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
+                 * going through the list of ports on the subnet
+                 */
                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
                 if (!sm.isPresent()) {
                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
-                }
-                Subnetmap subMap = sm.get();
-                List<Uuid> portList = subMap.getPortList();
-                if(portList!=null){
-                    InstanceIdentifier<PortOpData> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).build();
-                    Optional<PortOpData> optionalPortOp = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
-                    if(!optionalPortOp.isPresent()){
-                        logger.error("Stale ports removal: Cannot delete port. Not available in data store");
-                        return;
-                    } else{
-                        PortOpData portOpData = optionalPortOp.get();
-                        List<PortOpDataEntry> portOpDataList = portOpData.getPortOpDataEntry();
-                        if(portOpDataList!=null){
-                            for(PortOpDataEntry portOpDataListEntry :  portOpDataList){
-                                if(portList.contains(new Uuid(portOpDataListEntry.getPortId()))){
-                                    logger.trace("Removing stale port: " + portOpDataListEntry + "for dissociated subnetId: " + subnetId);
-                                    MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier.
-                                            child(PortOpDataEntry.class, new PortOpDataEntryKey(portOpDataListEntry.getKey())));
-                                }
-                            }
+                } else {
+                    Subnetmap subMap = sm.get();
+                    List<Uuid> portList = subMap.getPortList();
+                    if (portList != null) {
+                        for (Uuid port : portList) {
+                            InstanceIdentifier<PortOpDataEntry> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).
+                                    child(PortOpDataEntry.class, new PortOpDataEntryKey(port.getValue())).build();
+                            logger.trace("Deleting portOpData entry for port " + port.getValue());
+                            MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
                         }
                     }
                 }
 
+                SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
                 String rd = subOpBuilder.getVrfId();
                 String subnetIp = subOpBuilder.getSubnetCidr();
-                BigInteger nhDpnId = subOpBuilder.getNhDpnId();
+                String vpnName = subOpBuilder.getVpnName();
                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
                 logger.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore", subnetId.getValue());
                 try {
                     //Withdraw the routes for all the interfaces on this subnet
                     //Remove subnet route entry from FIB
-                    deleteSubnetRouteFromFib(rd, subnetIp);
+                    deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
                     withdrawSubnetRoutefromBgp(rd, subnetIp);
                 } catch (Exception ex) {
                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
@@ -322,7 +296,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         } else {
             if (notification.isExternalVpn()) {
                 SubnetAddedToVpnBuilder bldr = new SubnetAddedToVpnBuilder().setVpnName(vpnName).setElanTag(elanTag);
-                bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);
+                bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);;
                 onSubnetAddedToVpn(bldr.build());
             }
             // TODO(vivek): Something got updated, but we donot know what ?
@@ -339,7 +313,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         synchronized (this) {
             try {
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
-                        child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
+                    child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 
                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
@@ -454,12 +428,12 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                             subOpBuilder.setNhDpnId(null);
                             try {
                                 // withdraw route from BGP
-                                deleteSubnetRouteFromFib(rd, subnetIp);
+                                deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
                                 subOpBuilder.setRouteAdvState(TaskState.Na);
                             } catch (Exception ex) {
                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
-                                        subnetId.getValue() + " from BGP failed ", ex);
+                                  subnetId.getValue() + " from BGP failed ", ex);
                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
                             }
                         } else {
@@ -517,7 +491,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
             Uuid subnetId = portOpEntry.getSubnetId();
             try {
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
-                        child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
+                    child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (!optionalSubs.isPresent()) {
@@ -554,7 +528,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                         subOpBuilder.setRouteAdvState(TaskState.Done);
                     } catch (Exception ex) {
                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
-                                subnetId.getValue() + " to BGP failed {}" + ex);
+                          subnetId.getValue() + " to BGP failed {}" + ex);
                     }
                 }
                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
@@ -618,12 +592,12 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                             subOpBuilder.setNhDpnId(null);
                             try {
                                 // Withdraw route from BGP for this subnet
-                                deleteSubnetRouteFromFib(rd, subnetIp);
+                                deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
                                 subOpBuilder.setRouteAdvState(TaskState.Na);
                             } catch (Exception ex) {
                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
-                                        subnetId.getValue() + " from BGP failed {}" + ex);
+                                  subnetId.getValue() + " from BGP failed {}" + ex);
                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
                             }
                         } else {
@@ -656,30 +630,38 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         }
     }
 
+    @Override
+    public void onRouterAssociatedToVpn(RouterAssociatedToVpn notification) {
+    }
+
+    @Override
+    public void onRouterDisassociatedFromVpn(RouterDisassociatedFromVpn notification) {
+    }
+
     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
                                      Long elanTag, int label) {
         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
-        String nexthopIp = null;
-        if (nhDpnId != null) {
-            nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
-        }
-        vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, subnetIp, nexthopIp, label, elanTag);
+        String nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
+        if(nexthopIp != null)
+            vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, vpnName, subnetIp, nexthopIp, label, elanTag, nhDpnId);
+        else
+            logger.info("Unable to get nextHop ip address for nextHop DPN {}. Abort adding subnet route to FIB table.", nhDpnId);
     }
 
     private int getLabel(String rd, String subnetIp) {
         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
-                VpnUtil.getNextHopLabelKey(rd, subnetIp));
+                                        VpnUtil.getNextHopLabelKey(rd, subnetIp));
         logger.trace("Allocated subnetroute label {} for rd {} prefix {}", label, rd, subnetIp);
         return label;
     }
 
-    private void deleteSubnetRouteFromFib(String rd, String subnetIp) {
+    private void deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName) {
         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
-        vpnInterfaceManager.removeFibEntryFromDS(rd, subnetIp);
+        vpnInterfaceManager.deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
     }
 
     private void advertiseSubnetRouteToBgp(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
@@ -698,8 +680,8 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         try {
             // BGPManager (inside ODL) requires a withdraw followed by advertise
             // due to bugs with ClusterDataChangeListener used by BGPManager.
-            bgpManager.withdrawPrefix(rd, subnetIp);
-            bgpManager.advertisePrefix(rd, subnetIp, nexthopIp, label);
+            //bgpManager.withdrawPrefix(rd, subnetIp);
+            bgpManager.advertisePrefix(rd, subnetIp, Arrays.asList(nexthopIp), label);
         } catch (Exception e) {
             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
             throw e;
@@ -716,14 +698,5 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
             throw e;
         }
     }
-
-    @Override
-    public void onRouterAssociatedToVpn(RouterAssociatedToVpn notification) {
-    }
-
-    @Override
-    public void onRouterDisassociatedFromVpn(RouterDisassociatedFromVpn notification) {
-
-    }
 }
 
index 1d8f7980df11a61ce3d508174e541d3b2844c66c..32f0c72a7e070bf6d331ae67ecaa608850d515b8 100644 (file)
@@ -10,11 +10,12 @@ package org.opendaylight.netvirt.vpnmanager;
 
 import java.math.BigInteger;
 import java.net.InetAddress;
+import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
-import java.util.List;
 
 import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
 import com.google.common.primitives.Ints;
 import com.google.common.util.concurrent.CheckedFuture;
 import com.google.common.util.concurrent.FutureCallback;
@@ -25,7 +26,11 @@ import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
+import org.opendaylight.genius.mdsalutil.MDSALUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
+import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
+import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
@@ -37,13 +42,22 @@ import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev14081
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.Interfaces;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.InterfaceKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanTagNameMap;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.tag.name.map.ElanTagName;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.tag.name.map.ElanTagNameKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIds;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.VpnIdsKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder;
@@ -56,15 +70,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstanceBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.Vpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.VpnKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesListKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
@@ -83,12 +95,19 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.met
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.tag.name.map.ElanTagName;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanTagNameMap;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.tag.name.map.ElanTagNameKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronVpnPortipPortData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPortBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPortKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinkStates;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinks;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkStateKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.opendaylight.yangtools.yang.common.RpcResult;
 import org.opendaylight.yangtools.yang.binding.DataObject;
+import org.opendaylight.yangtools.yang.data.impl.schema.tree.SchemaValidationFailedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -107,75 +126,208 @@ public class VpnUtil {
                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
     }
 
-    static VpnInterface getVpnInterface(String intfName, String vpnName, Adjacencies aug) {
-        return new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(intfName)).setVpnInstanceName(vpnName)
-                .addAugmentation(Adjacencies.class, aug)
+    static VpnInterface getVpnInterface(String intfName, String vpnName, Adjacencies aug, BigInteger dpnId, Boolean isSheduledForRemove) {
+        return new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(intfName)).setVpnInstanceName(vpnName).setDpnId(dpnId)
+                .setScheduledForRemove(isSheduledForRemove).addAugmentation(Adjacencies.class, aug)
                 .build();
     }
 
     static InstanceIdentifier<Prefixes> getPrefixToInterfaceIdentifier(long vpnId, String ipPrefix) {
         return InstanceIdentifier.builder(PrefixToInterface.class)
-            .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class,
-                                                             new PrefixesKey(ipPrefix)).build();
+                .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class,
+                        new PrefixesKey(ipPrefix)).build();
+    }
+
+    static InstanceIdentifier<VpnIds> getPrefixToInterfaceIdentifier(long vpnId) {
+        return InstanceIdentifier.builder(PrefixToInterface.class)
+                .child(VpnIds.class, new VpnIdsKey(vpnId)).build();
+    }
+
+    static VpnIds getPrefixToInterface(long vpnId) {
+        return new VpnIdsBuilder().setKey(new VpnIdsKey(vpnId)).setVpnId(vpnId).build();
     }
 
     static Prefixes getPrefixToInterface(BigInteger dpId, String vpnInterfaceName, String ipPrefix) {
         return new PrefixesBuilder().setDpnId(dpId).setVpnInterfaceName(
-            vpnInterfaceName).setIpAddress(ipPrefix).build();
+                vpnInterfaceName).setIpAddress(ipPrefix).build();
     }
 
     static InstanceIdentifier<Extraroute> getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) {
         return InstanceIdentifier.builder(VpnToExtraroute.class)
                 .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class,
-                 new ExtrarouteKey(ipPrefix)).build();
+                        new ExtrarouteKey(ipPrefix)).build();
     }
 
-    static List<Adjacency> getAdjacenciesForVpnInterfaceFromConfig(DataBroker broker, String intfName) {
-        final InstanceIdentifier<VpnInterface> identifier = getVpnInterfaceIdentifier(intfName);
-        InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
-        Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
+    static InstanceIdentifier<Vpn> getVpnToExtrarouteIdentifier(String vrfId) {
+        return InstanceIdentifier.builder(VpnToExtraroute.class)
+                .child(Vpn.class, new VpnKey(vrfId)).build();
+    }
 
-        if (adjacencies.isPresent()) {
-            List<Adjacency> nextHops = adjacencies.get().getAdjacency();
-            return nextHops;
+    static Vpn getVpnToExtraRoute(String vrfId) {
+        return new VpnBuilder().setKey(new VpnKey(vrfId)).setVrfId(vrfId).build();
+    }
+
+    /**
+     * Retrieves the Instance Identifier that points to an InterVpnLink object
+     * in MDSL
+     *
+     * @param vpnLinkName The name of the InterVpnLink
+     * @return The requested InstanceIdentifier
+     */
+    public static InstanceIdentifier<InterVpnLinkState> getInterVpnLinkStateIid(String vpnLinkName) {
+        return InstanceIdentifier.builder(InterVpnLinkStates.class).child(InterVpnLinkState.class, new InterVpnLinkStateKey(vpnLinkName)).build();
+    }
+
+    /**
+     * Get inter-VPN link state
+     *
+     * @param broker dataBroker service reference
+     * @param vpnLinkName The name of the InterVpnLink
+     * @return the object that contains the State of the specified InterVpnLink
+     */
+    public static InterVpnLinkState getInterVpnLinkState(DataBroker broker, String vpnLinkName) {
+        InstanceIdentifier<InterVpnLinkState> vpnLinkStateIid = VpnUtil.getInterVpnLinkStateIid(vpnLinkName);
+        Optional<InterVpnLinkState> vpnLinkState = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION,
+                vpnLinkStateIid);
+        if (vpnLinkState.isPresent()) {
+            return vpnLinkState.get();
         }
         return null;
     }
 
-    static Extraroute getVpnToExtraroute(String ipPrefix, String nextHop) {
-        return new ExtrarouteBuilder().setPrefix(ipPrefix).setNexthopIp(nextHop).build();
+    /**
+     * Get VRF table given a Route Distinguisher
+     *
+     * @param broker dataBroker service reference
+     * @param rd Route-Distinguisher
+     * @return VrfTables that holds the list of VrfEntries of the specified rd
+     */
+    public static VrfTables getVrfTable(DataBroker broker, String rd) {
+        InstanceIdentifier<VrfTables> id =
+                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).build();
+        Optional<VrfTables> vrfTable = read(broker, LogicalDatastoreType.CONFIGURATION, id);
+        return vrfTable.isPresent() ? vrfTable.get() : null;
     }
 
-    static Adjacencies
-    getVpnInterfaceAugmentation(List<Adjacency> nextHops) {
-        return new AdjacenciesBuilder().setAdjacency(nextHops).build();
+    /**
+     * Retrieves the VrfEntries that belong to a given VPN filtered out by
+     * Origin, searching by its Route-Distinguisher
+     *
+     * @param broker dataBroker service reference
+     * @param rd     Route-distinguisher of the VPN
+     * @param originsToConsider Only entries whose origin is included in this
+     *     list will be considered
+     * @return the list of VrfEntries
+     */
+    public static List<VrfEntry> getVrfEntriesByOrigin(DataBroker broker, String rd,
+                                                       List<RouteOrigin> originsToConsider) {
+        List<VrfEntry> result = new ArrayList<VrfEntry>();
+        List<VrfEntry> allVpnVrfEntries = getAllVrfEntries(broker, rd);
+        for (VrfEntry vrfEntry : allVpnVrfEntries) {
+            if (originsToConsider.contains(RouteOrigin.value(vrfEntry.getOrigin()))) {
+                result.add(vrfEntry);
+            }
+        }
+        return result;
     }
 
-    public static InstanceIdentifier<IdPool> getPoolId(String poolName){
-        InstanceIdentifier.InstanceIdentifierBuilder<IdPool> idBuilder =
-                        InstanceIdentifier.builder(IdPools.class).child(IdPool.class, new IdPoolKey(poolName));
-        InstanceIdentifier<IdPool> id = idBuilder.build();
-        return id;
+    static List<Prefixes> getAllPrefixesToInterface(DataBroker broker, long vpnId) {
+        Optional<VpnIds> vpnIds = read(broker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId));
+        if (vpnIds.isPresent()) {
+            return vpnIds.get().getPrefixes();
+        }
+        return new ArrayList<Prefixes>();
+    }
+
+    static List<Extraroute> getAllExtraRoutes(DataBroker broker, String vrfId) {
+        Optional<Vpn> extraRoutes = read(broker, LogicalDatastoreType.OPERATIONAL, getVpnToExtrarouteIdentifier(vrfId));
+        if (extraRoutes.isPresent()) {
+            return extraRoutes.get().getExtraroute();
+        }
+        return new ArrayList<Extraroute>();
+    }
+
+    /**
+     * Retrieves all the VrfEntries that belong to a given VPN searching by its
+     * Route-Distinguisher
+     *
+     * @param broker dataBroker service reference
+     * @param rd     Route-distinguisher of the VPN
+     * @return the list of VrfEntries
+     */
+    public static List<VrfEntry> getAllVrfEntries(DataBroker broker, String rd) {
+        VrfTables vrfTables = VpnUtil.getVrfTable(broker, rd);
+        return (vrfTables != null) ? vrfTables.getVrfEntry() : new ArrayList<VrfEntry>();
+    }
+
+    //FIXME: Implement caches for DS reads
+    static VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
+        InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class).child(VpnInstance.class,
+                new VpnInstanceKey(vpnInstanceName)).build();
+        Optional<VpnInstance> vpnInstance = read(broker, LogicalDatastoreType.CONFIGURATION, id);
+        return (vpnInstance.isPresent()) ? vpnInstance.get() : null;
+    }
+
+    static List<VpnInstance> getAllVpnInstance(DataBroker broker) {
+        InstanceIdentifier<VpnInstances> id = InstanceIdentifier.builder(VpnInstances.class).build();
+        Optional<VpnInstances> optVpnInstances = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
+        if (optVpnInstances.isPresent()) {
+            return optVpnInstances.get().getVpnInstance();
+        } else {
+            return new ArrayList<VpnInstance>();
+        }
     }
 
     static VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) {
-        InstanceIdentifier.InstanceIdentifierBuilder<VrfTables> idBuilder =
-                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
-        InstanceIdentifier<VrfTables> id = idBuilder.build();
 
-        Optional<VrfTables> vrfTable = read(broker, LogicalDatastoreType.CONFIGURATION, id);
-        if (vrfTable.isPresent()) {
+        VrfTables vrfTable = getVrfTable(broker, rd);
+        // TODO: why check VrfTables if we later go for the specific VrfEntry?
+        if (vrfTable != null) {
             InstanceIdentifier<VrfEntry> vrfEntryId =
                     InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).
                             child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build();
             Optional<VrfEntry> vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
-            if (vrfEntry.isPresent())  {
-                return ((VrfEntry)vrfEntry.get());
+            if (vrfEntry.isPresent()) {
+                return (vrfEntry.get());
             }
         }
         return null;
     }
 
+    static List<Adjacency> getAdjacenciesForVpnInterfaceFromConfig(DataBroker broker, String intfName) {
+        final InstanceIdentifier<VpnInterface> identifier = getVpnInterfaceIdentifier(intfName);
+        InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
+        Optional<Adjacencies> adjacencies = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, path);
+
+        if (adjacencies.isPresent()) {
+            List<Adjacency> nextHops = adjacencies.get().getAdjacency();
+            return nextHops;
+        }
+        return null;
+    }
+
+    static Extraroute getVpnToExtraroute(String ipPrefix, List<String> nextHopList) {
+        return new ExtrarouteBuilder().setPrefix(ipPrefix).setNexthopIpList(nextHopList).build();
+    }
+
+    public static List<Extraroute> getVpnExtraroutes(DataBroker broker, String vpnRd) {
+        InstanceIdentifier<Vpn> vpnExtraRoutesId =
+                InstanceIdentifier.builder(VpnToExtraroute.class).child(Vpn.class, new VpnKey(vpnRd)).build();
+        Optional<Vpn> vpnOpc = read(broker, LogicalDatastoreType.OPERATIONAL, vpnExtraRoutesId);
+        return vpnOpc.isPresent() ? vpnOpc.get().getExtraroute() : new ArrayList<Extraroute>();
+    }
+
+    static Adjacencies getVpnInterfaceAugmentation(List<Adjacency> nextHopList) {
+        return new AdjacenciesBuilder().setAdjacency(nextHopList).build();
+    }
+
+    public static InstanceIdentifier<IdPool> getPoolId(String poolName) {
+        InstanceIdentifier.InstanceIdentifierBuilder<IdPool> idBuilder =
+                InstanceIdentifier.builder(IdPools.class).child(IdPool.class, new IdPoolKey(poolName));
+        InstanceIdentifier<IdPool> id = idBuilder.build();
+        return id;
+    }
+
     static InstanceIdentifier<VpnInterfaces> getVpnInterfacesIdentifier() {
         return InstanceIdentifier.builder(VpnInterfaces.class).build();
     }
@@ -187,36 +339,38 @@ public class VpnUtil {
 
     static InstanceIdentifier<VpnToDpnList> getVpnToDpnListIdentifier(String rd, BigInteger dpnId) {
         return InstanceIdentifier.builder(VpnInstanceOpData.class)
-            .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd))
-            .child(VpnToDpnList.class, new VpnToDpnListKey(dpnId)).build();
+                .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd))
+                .child(VpnToDpnList.class, new VpnToDpnListKey(dpnId)).build();
     }
 
     public static BigInteger getCookieArpFlow(int interfaceTag) {
         return VpnConstants.COOKIE_L3_BASE.add(new BigInteger("0110000", 16)).add(
-            BigInteger.valueOf(interfaceTag));
+                BigInteger.valueOf(interfaceTag));
+    }
+
+    public static BigInteger getCookieL3(int vpnId) {
+        return VpnConstants.COOKIE_L3_BASE.add(new BigInteger("0610000", 16)).add(BigInteger.valueOf(vpnId));
     }
 
     public static String getFlowRef(BigInteger dpnId, short tableId, int ethType, int lPortTag, int arpType) {
-            return new StringBuffer().append(VpnConstants.FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
-                    .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(ethType).append(lPortTag)
-                    .append(NwConstants.FLOWID_SEPARATOR).append(arpType).toString();
+        return new StringBuffer().append(VpnConstants.FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
+                .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(ethType).append(lPortTag)
+                .append(NwConstants.FLOWID_SEPARATOR).append(arpType).toString();
     }
 
-    public static int getUniqueId(IdManagerService idManager, String poolName,String idKey) {
-        AllocateIdInput getIdInput = new AllocateIdInputBuilder()
-                                           .setPoolName(poolName)
-                                           .setIdKey(idKey).build();
+    public static int getUniqueId(IdManagerService idManager, String poolName, String idKey) {
+        AllocateIdInput getIdInput = new AllocateIdInputBuilder().setPoolName(poolName).setIdKey(idKey).build();
 
         try {
             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
             RpcResult<AllocateIdOutput> rpcResult = result.get();
-            if(rpcResult.isSuccessful()) {
+            if (rpcResult.isSuccessful()) {
                 return rpcResult.getResult().getIdValue().intValue();
             } else {
                 LOG.warn("RPC Call to Get Unique Id returned with Errors {}", rpcResult.getErrors());
             }
         } catch (InterruptedException | ExecutionException e) {
-            LOG.warn("Exception when getting Unique Id",e);
+            LOG.warn("Exception when getting Unique Id", e);
         }
         return 0;
     }
@@ -226,7 +380,7 @@ public class VpnUtil {
         try {
             Future<RpcResult<Void>> result = idManager.releaseId(idInput);
             RpcResult<Void> rpcResult = result.get();
-            if(!rpcResult.isSuccessful()) {
+            if (!rpcResult.isSuccessful()) {
                 LOG.warn("RPC Call to Get Unique Id returned with Errors {}", rpcResult.getErrors());
             }
         } catch (InterruptedException | ExecutionException e) {
@@ -234,53 +388,78 @@ public class VpnUtil {
         }
     }
 
-    public static String getNextHopLabelKey(String rd, String prefix){
-        String key = rd + VpnConstants.SEPARATOR + prefix;
-        return key;
+    public static String getNextHopLabelKey(String rd, String prefix) {
+        return rd + VpnConstants.SEPARATOR + prefix;
+    }
+
+    /**
+     * Retrieves the VpnInstance name (typically the VPN Uuid) out from the
+     * route-distinguisher
+     *
+     * @param broker dataBroker service reference
+     * @param rd Route-Distinguisher
+     * @return the VpnInstance name
+     */
+    public static String getVpnNameFromRd(DataBroker broker, String rd) {
+        VpnInstanceOpDataEntry vpnInstanceOpData = getVpnInstanceOpData(broker, rd);
+        return (vpnInstanceOpData != null) ? vpnInstanceOpData.getVpnInstanceName() : null;
     }
 
+    /**
+     * Retrieves the dataplane identifier of a specific VPN, searching by its
+     * VpnInstance name.
+     *
+     * @param broker dataBroker service reference
+     * @param vpnName Name of the VPN
+     * @return the dataplane identifier of the VPN, the VrfTag.
+     */
     public static long getVpnId(DataBroker broker, String vpnName) {
 
         InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> id
-            = getVpnInstanceToVpnIdIdentifier(vpnName);
+                = getVpnInstanceToVpnIdIdentifier(vpnName);
         Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> vpnInstance
-            = read(broker, LogicalDatastoreType.CONFIGURATION, id);
+                = read(broker, LogicalDatastoreType.CONFIGURATION, id);
 
         long vpnId = VpnConstants.INVALID_ID;
-        if(vpnInstance.isPresent()) {
+        if (vpnInstance.isPresent()) {
             vpnId = vpnInstance.get().getVpnId();
         }
         return vpnId;
     }
 
+    /**
+     * Retrieves the VPN Route Distinguisher searching by its Vpn instance name
+     *
+     * @param broker dataBroker service reference
+     * @param vpnName Name of the VPN
+     * @return the route-distinguisher of the VPN
+     */
     public static String getVpnRd(DataBroker broker, String vpnName) {
-
         InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> id
                 = getVpnInstanceToVpnIdIdentifier(vpnName);
         Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance> vpnInstance
                 = read(broker, LogicalDatastoreType.CONFIGURATION, id);
 
         String rd = null;
-        if(vpnInstance.isPresent()) {
+        if (vpnInstance.isPresent()) {
             rd = vpnInstance.get().getVrfId();
         }
         return rd;
     }
 
-    //FIXME: Implement caches for DS reads
-    static VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
-        InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class).child(VpnInstance.class,
-                new VpnInstanceKey(vpnInstanceName)).build();
-        Optional<VpnInstance> vpnInstance = read(broker, LogicalDatastoreType.CONFIGURATION, id);
-        return (vpnInstance.isPresent()) ? vpnInstance.get() : null;
-    }
-
-    static String getVpnRdFromVpnInstanceConfig(DataBroker broker, String vpnName) {
+    /**
+     * Get VPN Route Distinguisher from VPN Instance Configuration
+     *
+     * @param broker dataBroker service reference
+     * @param vpnName Name of the VPN
+     * @return the route-distinguisher of the VPN
+     */
+    public static String getVpnRdFromVpnInstanceConfig(DataBroker broker, String vpnName) {
         InstanceIdentifier<VpnInstance> id = InstanceIdentifier.builder(VpnInstances.class)
                 .child(VpnInstance.class, new VpnInstanceKey(vpnName)).build();
         Optional<VpnInstance> vpnInstance = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, id);
         String rd = null;
-        if(vpnInstance.isPresent()) {
+        if (vpnInstance.isPresent()) {
             VpnInstance instance = vpnInstance.get();
             VpnAfConfig config = instance.getIpv4Family();
             rd = config.getRouteDistinguisher();
@@ -288,34 +467,200 @@ public class VpnUtil {
         return rd;
     }
 
+    /**
+     * Writes in MDSAL a new VrfEntry for a VPN specified by its Route-Distinguisher. It performs a merge,
+     * that is, if the VrfEntry exists the nexthop is added the VrfEntry nexthop list. If the entry does not
+     * exist, it is created.
+     *
+     * @param broker dataBroker service reference
+     * @param rd Route-distinguisher of the VPN
+     * @param prefix Destination of the route. This, together with the RD, identifies the VrfEntry
+     * @param nextHop Nexthop of the route
+     * @param label Label of the route
+     */
+    public static void addFibEntryToDS(DataBroker broker, String rd, String prefix, String nextHop, int label, RouteOrigin origin) {
+        Preconditions.checkNotNull(rd, "RD cannot be null");
+        LOG.debug("Created vrfEntry for {} nexthop {} label {}", prefix, nextHop, label);
+        synchronized (rd.intern()) {
+            InstanceIdentifier<VrfTables> vrfTableId =
+                    InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).build();
+            VrfEntry vrfEntry = getVrfEntry(broker, rd, prefix);
+            if (vrfEntry != null) {
+                List<String> nextHopList = vrfEntry.getNextHopAddressList();
+                nextHopList.add(nextHop);
+                VrfEntryBuilder builder = new VrfEntryBuilder(vrfEntry).setNextHopAddressList(nextHopList);
+                VrfEntry newVrfEntry = builder.build();
+                // Just update the VrfEntry
+                VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION,
+                        vrfTableId.child(VrfEntry.class, new VrfEntryKey(prefix)), newVrfEntry);
+            } else {
+                List<VrfEntry> currentVrfEntries = new ArrayList<VrfEntry>();
+                VrfEntryBuilder builder = new VrfEntryBuilder().setDestPrefix(prefix).setNextHopAddressList(Arrays.asList(nextHop))
+                        .setLabel((long) label).setOrigin(origin.getValue());
+                vrfEntry = builder.build();
+                currentVrfEntries.add(vrfEntry);
+                VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(
+                        currentVrfEntries).build();
+                VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
+            }
+            LOG.info("ADD: Added Fib Entry rd {} prefix {} nextHop {} label {}", rd, prefix, nextHop, label);
+        }
+
+    }
+
+    /**
+     * Removes a specific Nexthop from a VrfEntry. If Nexthop to remove is the
+     * last one in the VrfEntry, then the VrfEntry is removed too.
+     *
+     * @param broker dataBroker service reference
+     * @param rd Route-Distinguisher to which the VrfEntry belongs to
+     * @param prefix Destination of the route
+     * @param nextHopToRemove Specific nexthop within the Route to be removed.
+     *           If null or empty, then the whole VrfEntry is removed
+     */
+    public static void removeFibEntryFromDS(DataBroker broker, String rd, String prefix, String nextHopToRemove) {
+
+        LOG.debug("Removing fib entry with destination prefix {} from vrf table for rd {}", prefix, rd);
+
+        // Looking for existing prefix in MDSAL database
+        InstanceIdentifier<VrfEntry> vrfEntryId =
+            InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
+                                                        .child(VrfEntry.class, new VrfEntryKey(prefix)).build();
+        Optional<VrfEntry> entry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
+
+        if ( entry.isPresent() ) {
+            synchronized (rd.intern()) {
+                List<String> nhListRead = new ArrayList<>();
+                if ( nextHopToRemove != null && !nextHopToRemove.isEmpty()) {
+                    nhListRead = entry.get().getNextHopAddressList();
+                    if (nhListRead.contains(nextHopToRemove)) {
+                        nhListRead.remove(nextHopToRemove);
+                    }
+                }
+
+                if (nhListRead.isEmpty()) {
+                    // Remove the whole entry
+                    MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
+                    LOG.info("Removed Fib Entry rd {} prefix {}", rd, prefix);
+                } else {
+                    // An update must be done, not including the current next hop
+                    VrfEntry vrfEntry =
+                        new VrfEntryBuilder(entry.get()).setDestPrefix(prefix).setNextHopAddressList(nhListRead)
+                                                        .setKey(new VrfEntryKey(prefix)).build();
+                    MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
+                    MDSALUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId, vrfEntry);
+                    LOG.info("Removed Nexthop {} from Fib Entry rd {} prefix {}", nextHopToRemove, rd, prefix);
+                }
+            }
+        } else {
+            LOG.warn("Could not find VrfEntry for Route-Distinguisher={} and prefix={}", rd, prefix);
+        }
+    }
+
+
+    /**
+     * Remove from MDSAL all those VrfEntries in a VPN that have an specific RouteOrigin
+     *
+     * @param broker dataBroker service reference
+     * @param rd     Route Distinguisher
+     * @param origin Origin of the Routes to be removed (see {@link org.opendaylight.netvirt.bgpmanager.api.RouteOrigin})
+     */
+    public static void removeVrfEntriesByOrigin(DataBroker broker, String rd, RouteOrigin origin) {
+        InstanceIdentifier<VrfTables> vpnVrfTableIid =
+                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).build();
+        Optional<VrfTables> vrfTablesOpc = read(broker, LogicalDatastoreType.CONFIGURATION, vpnVrfTableIid);
+        if (vrfTablesOpc.isPresent()) {
+            VrfTables vrfTables = vrfTablesOpc.get();
+            List<VrfEntry> newVrfEntries = new ArrayList<VrfEntry>();
+            for (VrfEntry vrfEntry : vrfTables.getVrfEntry()) {
+                if (origin == RouteOrigin.value(vrfEntry.getOrigin())) {
+                    delete(broker, LogicalDatastoreType.CONFIGURATION, vpnVrfTableIid.child(VrfEntry.class,
+                            vrfEntry.getKey()));
+                }
+            }
+        }
+    }
+
+    public static void removeVrfEntriesByNexthop(DataBroker broker, String rd, String nexthop) {
+        InstanceIdentifier<VrfTables> vpnVrfTableIid =
+                InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)).build();
+        Optional<VrfTables> vrfTablesOpc = read(broker, LogicalDatastoreType.CONFIGURATION, vpnVrfTableIid);
+        if (vrfTablesOpc.isPresent()) {
+            VrfTables vrfTables = vrfTablesOpc.get();
+            for (VrfEntry vrfEntry : vrfTables.getVrfEntry()) {
+                if (vrfEntry.getNextHopAddressList() != null && vrfEntry.getNextHopAddressList().contains(nexthop)) {
+                    // TODO: Removes all the VrfEntry if one of the nexthops is the specified nexthop
+                    //                should we only remove the specific nexthop, or all the VrfEnry?
+                    delete(broker, LogicalDatastoreType.CONFIGURATION, vpnVrfTableIid.child(VrfEntry.class,
+                            vrfEntry.getKey()));
+                }
+            }
+        }
+    }
+
     static org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance
-           getVpnInstanceToVpnId(String vpnName, long vpnId, String rd) {
-        return new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstanceBuilder()
-            .setVpnId(vpnId).setVpnInstanceName(vpnName).setVrfId(rd).build();
+                getVpnInstanceToVpnId(String vpnName, long vpnId, String rd) {
+
+        return new VpnInstanceBuilder().setVpnId(vpnId).setVpnInstanceName(vpnName).setVrfId(rd).build();
 
     }
 
     static InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance>
-           getVpnInstanceToVpnIdIdentifier(String vpnName) {
+                getVpnInstanceToVpnIdIdentifier(String vpnName) {
         return InstanceIdentifier.builder(VpnInstanceToVpnId.class)
-            .child(org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance.class,
-                   new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstanceKey(vpnName)).build();
+                .child(org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance.class,
+                        new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstanceKey(vpnName)).build();
+    }
+
+    static org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds
+                getVpnIdToVpnInstance(long vpnId, String vpnName, String rd, boolean isExternalVpn) {
+        return new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIdsBuilder()
+                .setVpnId(vpnId).setVpnInstanceName(vpnName).setVrfId(rd).setExternalVpn(isExternalVpn).build();
+
+    }
+
+    static InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds>
+        getVpnIdToVpnInstanceIdentifier(long vpnId) {
+        return InstanceIdentifier.builder(VpnIdToVpnInstance.class)
+                .child(org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds.class,
+                        new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIdsKey(Long.valueOf(vpnId))).build();
     }
 
-    static InstanceIdentifier<VpnInstanceOpDataEntry> getVpnInstanceOpDataIdentifier(String rd) {
+    /**
+     * Retrieves the Vpn Name searching by its VPN Tag.
+     *
+     * @param broker dataBroker service reference
+     * @param vpnId Dataplane identifier of the VPN
+     * @return the Vpn instance name
+     */
+    public static String getVpnName(DataBroker broker, long vpnId) {
+
+        InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds> id
+                = getVpnIdToVpnInstanceIdentifier(vpnId);
+        Optional<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds> vpnInstance
+                = read(broker, LogicalDatastoreType.CONFIGURATION, id);
+
+        String vpnName = null;
+        if (vpnInstance.isPresent()) {
+            vpnName = vpnInstance.get().getVpnInstanceName();
+        }
+        return vpnName;
+    }
+
+    public static InstanceIdentifier<VpnInstanceOpDataEntry> getVpnInstanceOpDataIdentifier(String rd) {
         return InstanceIdentifier.builder(VpnInstanceOpData.class)
-            .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd)).build();
+                .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(rd)).build();
     }
 
-    static VpnInstanceOpDataEntry getVpnInstanceOpDataBuilder(String rd, long vpnId) {
-        return new VpnInstanceOpDataEntryBuilder().setVrfId(rd).setVpnId(vpnId).build();
+    static VpnInstanceOpDataEntry getVpnInstanceOpDataBuilder(String rd, long vpnId, String vpnName) {
+        return new VpnInstanceOpDataEntryBuilder().setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnName).build();
     }
 
-    static VpnInstanceOpDataEntry updateIntfCntInVpnInstOpData(Long count, String vrfId) {
-        return new VpnInstanceOpDataEntryBuilder().setVpnInterfaceCount(count).setVrfId(vrfId).build();
+    static VpnInstanceOpDataEntry updateIntfCntInVpnInstOpData(Long newCount, String vrfId) {
+        return new VpnInstanceOpDataEntryBuilder().setVpnInterfaceCount(newCount).setVrfId(vrfId).build();
     }
 
-       static InstanceIdentifier<RouterInterface> getRouterInterfaceId(String interfaceName) {
+    static InstanceIdentifier<RouterInterface> getRouterInterfaceId(String interfaceName) {
         return InstanceIdentifier.builder(RouterInterfaces.class)
                 .child(RouterInterface.class, new RouterInterfaceKey(interfaceName)).build();
     }
@@ -324,11 +669,11 @@ public class VpnUtil {
         return new RouterInterfaceBuilder().setKey(new RouterInterfaceKey(interfaceName))
                 .setInterfaceName(interfaceName).setRouterName(routerName).build();
     }
-       
+
     static VpnInstanceOpDataEntry getVpnInstanceOpData(DataBroker broker, String rd) {
         InstanceIdentifier<VpnInstanceOpDataEntry> id = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
         Optional<VpnInstanceOpDataEntry> vpnInstanceOpData = read(broker, LogicalDatastoreType.OPERATIONAL, id);
-        if(vpnInstanceOpData.isPresent()) {
+        if (vpnInstanceOpData.isPresent()) {
             return vpnInstanceOpData.get();
         }
         return null;
@@ -354,8 +699,7 @@ public class VpnUtil {
         return null;
     }
 
-    static boolean isVpnInterfaceConfigured(DataBroker broker, String interfaceName)
-    {
+    static boolean isVpnInterfaceConfigured(DataBroker broker, String interfaceName) {
         InstanceIdentifier<VpnInterface> interfaceId = getVpnInterfaceIdentifier(interfaceName);
         Optional<VpnInterface> configuredVpnInterface = read(broker, LogicalDatastoreType.CONFIGURATION, interfaceId);
 
@@ -368,24 +712,28 @@ public class VpnUtil {
     static String getIpPrefix(String prefix) {
         String prefixValues[] = prefix.split("/");
         if (prefixValues.length == 1) {
-            prefix = prefix + PREFIX_SEPARATOR + DEFAULT_PREFIX_LENGTH ;
+            prefix = prefix + PREFIX_SEPARATOR + DEFAULT_PREFIX_LENGTH;
         }
         return prefix;
     }
 
     static final FutureCallback<Void> DEFAULT_CALLBACK =
             new FutureCallback<Void>() {
+                @Override
                 public void onSuccess(Void result) {
                     LOG.debug("Success in Datastore operation");
                 }
 
+                @Override
                 public void onFailure(Throwable error) {
                     LOG.error("Error in Datastore operation", error);
                 }
+
+                ;
             };
 
     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
-                                                    InstanceIdentifier<T> path) {
+                                                          InstanceIdentifier<T> path) {
 
         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 
@@ -401,29 +749,56 @@ public class VpnUtil {
         return result;
     }
 
-    static <T extends DataObject> void asyncUpdate(DataBroker broker, LogicalDatastoreType datastoreType,
-                                                      InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
+    public static <T extends DataObject> void asyncUpdate(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                          InstanceIdentifier<T> path, T data) {
+        asyncUpdate(broker, datastoreType, path, data, DEFAULT_CALLBACK);
+    }
+
+    public static <T extends DataObject> void asyncUpdate(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                          InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
         WriteTransaction tx = broker.newWriteOnlyTransaction();
         tx.merge(datastoreType, path, data, true);
         Futures.addCallback(tx.submit(), callback);
     }
 
-    static <T extends DataObject> void asyncWrite(DataBroker broker, LogicalDatastoreType datastoreType,
-                                                   InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
+    public static <T extends DataObject> void asyncWrite(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                         InstanceIdentifier<T> path, T data) {
+        asyncWrite(broker, datastoreType, path, data, DEFAULT_CALLBACK);
+    }
+
+    public static <T extends DataObject> void asyncWrite(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                         InstanceIdentifier<T> path, T data, FutureCallback<Void> callback) {
         WriteTransaction tx = broker.newWriteOnlyTransaction();
         tx.put(datastoreType, path, data, true);
         Futures.addCallback(tx.submit(), callback);
     }
 
-    static <T extends DataObject> void delete(DataBroker broker, LogicalDatastoreType datastoreType,
-                                               InstanceIdentifier<T> path, FutureCallback<Void> callback) {
+    public static <T extends DataObject> void tryDelete(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                     InstanceIdentifier<T> path) {
+        try {
+            delete(broker, datastoreType, path, DEFAULT_CALLBACK);
+        } catch ( SchemaValidationFailedException sve ) {
+            LOG.info("Could not delete {}. SchemaValidationFailedException: {}", path, sve.getMessage());
+        } catch ( Exception e) {
+            LOG.info("Could not delete {}. Unhandled error: {}", path, e.getMessage());
+        }
+    }
+
+    public static <T extends DataObject> void delete(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                     InstanceIdentifier<T> path) {
+        delete(broker, datastoreType, path, DEFAULT_CALLBACK);
+    }
+
+
+    public static <T extends DataObject> void delete(DataBroker broker, LogicalDatastoreType datastoreType,
+                                                     InstanceIdentifier<T> path, FutureCallback<Void> callback) {
         WriteTransaction tx = broker.newWriteOnlyTransaction();
         tx.delete(datastoreType, path);
         Futures.addCallback(tx.submit(), callback);
     }
 
     public static <T extends DataObject> void syncWrite(DataBroker broker, LogicalDatastoreType datastoreType,
-                    InstanceIdentifier<T> path, T data) {
+                                                        InstanceIdentifier<T> path, T data) {
         WriteTransaction tx = broker.newWriteOnlyTransaction();
         tx.put(datastoreType, path, data, true);
         CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
@@ -436,7 +811,7 @@ public class VpnUtil {
     }
 
     public static <T extends DataObject> void syncUpdate(DataBroker broker, LogicalDatastoreType datastoreType,
-                    InstanceIdentifier<T> path, T data) {
+                                                         InstanceIdentifier<T> path, T data) {
         WriteTransaction tx = broker.newWriteOnlyTransaction();
         tx.merge(datastoreType, path, data, true);
         CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
@@ -449,14 +824,14 @@ public class VpnUtil {
     }
 
     public static long getRemoteBCGroup(long elanTag) {
-        return VpnConstants.ELAN_GID_MIN + ((elanTag % VpnConstants.ELAN_GID_MIN) *2);
+        return VpnConstants.ELAN_GID_MIN + ((elanTag % VpnConstants.ELAN_GID_MIN) * 2);
     }
 
     // interface-index-tag operational container
     public static IfIndexInterface getInterfaceInfoByInterfaceTag(DataBroker broker, long interfaceTag) {
         InstanceIdentifier<IfIndexInterface> interfaceId = getInterfaceInfoEntriesOperationalDataPath(interfaceTag);
         Optional<IfIndexInterface> existingInterfaceInfo = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, interfaceId);
-        if(existingInterfaceInfo.isPresent()) {
+        if (existingInterfaceInfo.isPresent()) {
             return existingInterfaceInfo.get();
         }
         return null;
@@ -467,27 +842,10 @@ public class VpnUtil {
                 new IfIndexInterfaceKey((int) interfaceTag)).build();
     }
 
-    // FIXME 1: To be fixed with VPNManager patch
-    /*public static String getNeutronPortNamefromPortFixedIp(DataBroker broker, String fixedIp) {
-        InstanceIdentifier id = buildFixedIpToPortNameIdentifier(fixedIp);
-        Optional<PortFixedipToPortName> portFixedipToPortNameData = read(broker, LogicalDatastoreType.CONFIGURATION,
-                id);
-        if (portFixedipToPortNameData.isPresent()) {
-            return portFixedipToPortNameData.get().getPortName();
-        }
-        return null;
-    }
-
-    private static InstanceIdentifier<PortFixedipToPortName> buildFixedIpToPortNameIdentifier(String fixedIp) {
-        InstanceIdentifier<PortFixedipToPortName> id = InstanceIdentifier.builder(NeutronPortData.class).child
-                (PortFixedipToPortName.class, new PortFixedipToPortNameKey(fixedIp)).build();
-        return id;
-    }*/
-
-    public static ElanTagName getElanInfoByElanTag(DataBroker broker,long elanTag) {
+    public static ElanTagName getElanInfoByElanTag(DataBroker broker, long elanTag) {
         InstanceIdentifier<ElanTagName> elanId = getElanInfoEntriesOperationalDataPath(elanTag);
         Optional<ElanTagName> existingElanInfo = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, elanId);
-        if(existingElanInfo.isPresent()) {
+        if (existingElanInfo.isPresent()) {
             return existingElanInfo.get();
         }
         return null;
@@ -499,6 +857,7 @@ public class VpnUtil {
     }
 
 
+    // TODO: Move this to NwUtil
     public static boolean isIpInSubnet(int ipAddress, String subnetCidr) {
         String[] subSplit = subnetCidr.split("/");
         if (subSplit.length < 2) {
@@ -522,6 +881,99 @@ public class VpnUtil {
         return false;
     }
 
+    /**
+     * Returns the Path identifier to reach a specific interface in a specific DPN in a given VpnInstance
+     *
+     * @param vpnRd     Route-Distinguisher of the VpnInstance
+     * @param dpnId     Id of the DPN where the interface is
+     * @param ifaceName Interface name
+     * @return the Instance Identifier
+     */
+    public static InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+            .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces>
+    getVpnToDpnInterfacePath(String vpnRd, BigInteger dpnId, String ifaceName) {
+
+        return
+                InstanceIdentifier.builder(VpnInstanceOpData.class)
+                        .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(vpnRd))
+                        .child(VpnToDpnList.class, new VpnToDpnListKey(dpnId))
+                        .child(org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                                .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces.class,
+                                new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                                        .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey(ifaceName))
+                        .build();
+    }
+
+    /**
+     * Includes a DPN with the corresponding interface names in the VpnToDpn operational data.
+     * This method is preferably over mergeDpnInVpnToDpnMap(DataBroker, String, String, BigInteger, List)
+     * when there are several DPNs to be merged since it saves some readings from MDSAL.
+     *
+     * @param broker     dataBroker service reference
+     * @param vpnOpData  Reference to the object that holds the Operational data of the VpnInstance
+     * @param dpnId      Id of the DPN where the interfaces to be added to Operational data are located
+     * @param ifaceNames List of interface names
+     */
+    public static void mergeDpnInVpnToDpnMap(DataBroker broker, VpnInstanceOpDataEntry vpnOpData, BigInteger dpnId,
+                                             List<String> ifaceNames) {
+        Preconditions.checkNotNull(vpnOpData);
+        Preconditions.checkNotNull(ifaceNames);
+
+        for (String ifaceName : ifaceNames) {
+            InstanceIdentifier<org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                    .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces> vpnDpnIfaceIid =
+                    getVpnToDpnInterfacePath(vpnOpData.getKey().getVrfId(), dpnId, ifaceName);
+
+            org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                    .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfaces vpnDpnIface =
+                    new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                            .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list
+                            .VpnInterfacesBuilder().setKey(new org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn
+                            .instance.op.data.vpn.instance.op.data.entry.vpn.to.dpn.list.VpnInterfacesKey(ifaceName))
+                            .setInterfaceName(ifaceName)
+                            .build();
+
+            syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, vpnDpnIfaceIid, vpnDpnIface);
+        }
+    }
+
+    /**
+     * Includes a DPN with the corresponding interface names in the VpnToDpn operational data.
+     *
+     * @param broker dataBroker service reference
+     * @param vpnName Name of the VPN
+     * @param rd Route-Distinguisher
+     * @param dpnId Id of the DPN that includes the list of Ifaces to be
+     *           included in the Map
+     * @param ifaceNames List of interfaces to be included in the Map
+     */
+    public static void mergeDpnInVpnToDpnMap(DataBroker broker, String vpnName, String rd, BigInteger dpnId,
+                                             List<String> ifaceNames) {
+        InstanceIdentifier<VpnInstanceOpDataEntry> id = InstanceIdentifier.builder(VpnInstanceOpData.class)
+                .child(VpnInstanceOpDataEntry.class, new VpnInstanceOpDataEntryKey(vpnName))
+                .build();
+
+        Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
+            MDSALUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
+        if (vpnInstanceOpData.isPresent()) {
+            mergeDpnInVpnToDpnMap(broker, vpnInstanceOpData.get(), dpnId, ifaceNames);
+        }
+    }
+
+    /**
+     * Removes a specific interface from the VpnToDpn operative map.
+     *
+     * @param broker    dataBroker service reference
+     * @param rd        Route-distinguisher of the VPN
+     * @param dpnId     Id of the DPN where the interface is
+     * @param ifaceName interface name.
+     */
+    public static void removeIfaceFromVpnToDpnMap(DataBroker broker, String rd, BigInteger dpnId, String ifaceName) {
+        tryDelete(broker, LogicalDatastoreType.CONFIGURATION, getVpnToDpnInterfacePath(rd, dpnId, ifaceName));
+        // Note: tryDelete is a best-effort. Sometimes we want to update the VpnToDpnMap ifaces when the
+        // DPN has gone down (and the VpnToDpnMap has been removed in a different Thread)
+    }
+
     public static void removePrefixToInterfaceForVpnId(DataBroker broker, long vpnId) {
         try {
             // Clean up PrefixToInterface Operational DS
@@ -563,6 +1015,15 @@ public class VpnUtil {
         }
     }
 
+    public static void removeVpnIdToVpnInstance(DataBroker broker, long vpnId) {
+        try {
+            delete(broker, LogicalDatastoreType.CONFIGURATION, getVpnIdToVpnInstanceIdentifier(vpnId),
+                    DEFAULT_CALLBACK);
+        } catch (Exception e) {
+            LOG.error("Exception during clean up of VpnIdToVpnInstance for VPNID {}", vpnId, e);
+        }
+    }
+
     public static void removeVrfTableForVpn(DataBroker broker, String vpnName) {
         // Clean up FIB Entries Config DS
         try {
@@ -584,4 +1045,279 @@ public class VpnUtil {
             LOG.error("Exception during cleanup of L3NextHop for VPN ID {}", vpnId, e);
         }
     }
+
+    /**
+     * Retrieves all configured InterVpnLinks
+     *
+     * @param broker dataBroker service reference
+     * @return the list of InterVpnLinks
+     */
+    public static List<InterVpnLink> getAllInterVpnLinks(DataBroker broker) {
+        InstanceIdentifier<InterVpnLinks> interVpnLinksIid = InstanceIdentifier.builder(InterVpnLinks.class).build();
+
+        Optional<InterVpnLinks> interVpnLinksOpData = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION,
+                interVpnLinksIid);
+
+        return (interVpnLinksOpData.isPresent()) ? interVpnLinksOpData.get().getInterVpnLink()
+                : new ArrayList<InterVpnLink>();
+    }
+
+    /**
+     * Retrieves the list of DPNs where the endpoint of a VPN in an InterVPNLink was instantiated
+     *
+     * @param broker dataBroker service reference
+     * @param vpnLinkName the name of the InterVpnLink
+     * @param vpnUuid UUID of the VPN whose endpoint to be checked
+     * @return the list of DPN Ids
+     */
+    public static List<BigInteger> getVpnLinkEndpointDPNs(DataBroker broker, String vpnLinkName, String vpnUuid) {
+        InterVpnLinkState interVpnLinkState = getInterVpnLinkState(broker, vpnLinkName);
+        if (interVpnLinkState.getFirstEndpointState().getVpnUuid().getValue().equals(vpnUuid)) {
+            return interVpnLinkState.getFirstEndpointState().getDpId();
+        } else {
+            return interVpnLinkState.getSecondEndpointState().getDpId();
+        }
+    }
+
+    /**
+     * Retrieves an InterVpnLink by searching by one of its endpoint's IP.
+     *
+     * @param broker dataBroker service reference
+     * @param endpointIp IP to serch for.
+     * @return the InterVpnLink or null if no InterVpnLink can be found
+     */
+    public static InterVpnLink getInterVpnLinkByEndpointIp(DataBroker broker, String endpointIp) {
+        List<InterVpnLink> allInterVpnLinks = getAllInterVpnLinks(broker);
+        for (InterVpnLink interVpnLink : allInterVpnLinks) {
+            if (interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(endpointIp)
+                    || interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(endpointIp)) {
+                return interVpnLink;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Retrieves the InterVpnLink that has one of its 2 endpoints installed in
+     * the specified DpnId
+     *
+     * @param broker dataBroker service reference
+     * @param dpnId Id of the DPN
+     * @return The InterVpnLink object if found, Optional.absent() otherwise
+     */
+    public static Optional<InterVpnLink> getInterVpnLinkByDpnId(DataBroker broker, BigInteger dpnId) {
+        List<InterVpnLink> allInterVpnLinks = getAllInterVpnLinks(broker);
+        for (InterVpnLink interVpnLink : allInterVpnLinks) {
+            InterVpnLinkState interVpnLinkState = getInterVpnLinkState(broker, interVpnLink.getName());
+            if ( ( interVpnLinkState != null )
+                 && ( interVpnLinkState.getFirstEndpointState().getDpId().contains(dpnId)
+                      || interVpnLinkState.getSecondEndpointState().getDpId().contains(dpnId) ) ) {
+                return Optional.fromNullable(interVpnLink);
+            }
+        }
+        return Optional.absent();
+    }
+
+    /**
+     * Leaks a route from one VPN to another. By default, the origin for this leaked route is INTERVPN
+     *
+     * @param broker           dataBroker service reference
+     * @param bgpManager       Used to advertise routes to the BGP Router
+     * @param interVpnLink     Reference to the object that holds the info about the link between the 2 VPNs
+     * @param srcVpnUuid       UUID of the VPN that has the route that is going to be leaked to the other VPN
+     * @param dstVpnUuid       UUID of the VPN that is going to receive the route
+     * @param prefix           Prefix of the route
+     * @param label            Label of the route in the original VPN
+     */
+    public static void leakRoute(DataBroker broker, IBgpManager bgpManager, InterVpnLink interVpnLink,
+                                 String srcVpnUuid, String dstVpnUuid, String prefix, Long label) {
+        leakRoute(broker, bgpManager, interVpnLink, srcVpnUuid, dstVpnUuid, prefix, label, RouteOrigin.INTERVPN);
+    }
+
+    /**
+     * Leaks a route from one VPN to another.
+     *
+     * @param broker           dataBroker service reference
+     * @param bgpManager       Used to advertise routes to the BGP Router
+     * @param interVpnLink     Reference to the object that holds the info about the link between the 2 VPNs
+     * @param srcVpnUuid       UUID of the VPN that has the route that is going to be leaked to the other VPN
+     * @param dstVpnUuid       UUID of the VPN that is going to receive the route
+     * @param prefix           Prefix of the route
+     * @param label            Label of the route in the original VPN
+     * @param forcedOrigin     By default, origin for leaked routes should be INTERVPN, however it is possible to
+     *                         provide a different origin if desired.
+     */
+    public static void leakRoute(DataBroker broker, IBgpManager bgpManager, InterVpnLink interVpnLink,
+                                 String srcVpnUuid, String dstVpnUuid, String prefix, Long label,
+                                 RouteOrigin forcedOrigin) {
+        Preconditions.checkNotNull(interVpnLink);
+
+        // The source VPN must participate in the InterVpnLink
+        Preconditions.checkArgument(interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(srcVpnUuid)
+                        || interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(srcVpnUuid),
+                "The source VPN {} does not participate in the interVpnLink {}",
+                srcVpnUuid, interVpnLink.getName());
+        // The destination VPN must participate in the InterVpnLink
+        Preconditions.checkArgument(interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(dstVpnUuid)
+                        || interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(dstVpnUuid),
+                "The destination VPN {} does not participate in the interVpnLink {}",
+                dstVpnUuid, interVpnLink.getName());
+
+        boolean destinationIs1stEndpoint = interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(dstVpnUuid);
+
+        String endpointIp = (destinationIs1stEndpoint) ? interVpnLink.getSecondEndpoint().getIpAddress().getValue()
+                : interVpnLink.getFirstEndpoint().getIpAddress().getValue();
+
+        VrfEntry newVrfEntry = new VrfEntryBuilder().setKey(new VrfEntryKey(prefix)).setDestPrefix(prefix)
+                .setLabel(label).setNextHopAddressList(Arrays.asList(endpointIp))
+                .setOrigin(RouteOrigin.INTERVPN.getValue())
+                .build();
+
+        String dstVpnRd = getVpnRd(broker, dstVpnUuid);
+        InstanceIdentifier<VrfEntry> newVrfEntryIid =
+                InstanceIdentifier.builder(FibEntries.class)
+                        .child(VrfTables.class, new VrfTablesKey(dstVpnRd))
+                        .child(VrfEntry.class, new VrfEntryKey(newVrfEntry.getDestPrefix()))
+                        .build();
+        asyncWrite(broker, LogicalDatastoreType.CONFIGURATION, newVrfEntryIid, newVrfEntry);
+
+        // Finally, route is advertised it to the DC-GW. But while in the FibEntries the nexthop is the other
+        // endpoint's IP, in the DC-GW the nexthop for those prefixes are the IPs of those DPNs where the target
+        // VPN has been instantiated
+        List<String> ecmpNexthops = new ArrayList<String>();
+        InterVpnLinkState vpnLinkState = getInterVpnLinkState(broker, interVpnLink.getName());
+        List<BigInteger> dpnIdList = (destinationIs1stEndpoint) ? vpnLinkState.getFirstEndpointState().getDpId()
+                : vpnLinkState.getSecondEndpointState().getDpId();
+        List<String> nexthops = new ArrayList<String>();
+        for (BigInteger dpnId : dpnIdList) {
+            nexthops.add(InterfaceUtils.getEndpointIpAddressForDPN(broker, dpnId));
+        }
+        try {
+            bgpManager.advertisePrefix(dstVpnRd, newVrfEntry.getDestPrefix(), nexthops, label.intValue());
+        } catch (Exception exc) {
+            LOG.error("Could not advertise prefix {} with label {} to VPN rd={}",
+                    newVrfEntry.getDestPrefix(), label.intValue(), dstVpnRd);
+        }
+    }
+
+
+    /**
+     * Retrieves the ids of the currently operative DPNs
+     *
+     * @param dataBroker dataBroker service reference
+     * @return the list of DPNs currently operative
+     */
+    public static List<BigInteger> getOperativeDPNs(DataBroker dataBroker) {
+        List<BigInteger> result = new LinkedList<BigInteger>();
+        InstanceIdentifier<Nodes> nodesInstanceIdentifier = InstanceIdentifier.builder(Nodes.class).build();
+        Optional<Nodes> nodesOptional = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
+                nodesInstanceIdentifier);
+        if (!nodesOptional.isPresent()) {
+            return result;
+        }
+        Nodes nodes = nodesOptional.get();
+        List<Node> nodeList = nodes.getNode();
+        for (Node node : nodeList) {
+            NodeId nodeId = node.getId();
+            if (nodeId != null) {
+                BigInteger dpnId = MDSALUtil.getDpnIdFromNodeName(nodeId);
+                result.add(dpnId);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Retrieves a list of randomly selected DPNs, as many as specified.
+     *
+     * @param dataBroker dataBroker service reference
+     * @param numberOfDPNs Specifies how many Operative DPNs must be found
+     * @param excludingDPNs Specifies a blacklist of DPNs
+     * @return the list of DPN Ids
+     */
+    public static List<BigInteger> pickRandomDPNs(DataBroker dataBroker, int numberOfDPNs,
+                                                  List<BigInteger> excludingDPNs) {
+        List<BigInteger> dpnIdPool = getOperativeDPNs(dataBroker);
+        int poolSize = dpnIdPool.size();
+        if (poolSize <= numberOfDPNs) {
+            // You requested more than there is, I give you all I have.
+            return dpnIdPool;
+        }
+
+        // Random reorder
+        Collections.shuffle(dpnIdPool);
+        List<BigInteger> result = new ArrayList<BigInteger>();
+
+        for (BigInteger dpId : dpnIdPool) {
+            if (excludingDPNs == null || !excludingDPNs.contains(dpId)) {
+                result.add(dpId);
+                if (result.size() == numberOfDPNs)
+                    break;
+            }
+        }
+
+        if (result.size() < numberOfDPNs) {
+            // We still don't have all we need, so we have to pick up among the "prohibited" ones
+            dpnIdPool.removeAll(result);
+
+            int nbrOfProhibitedDpnsToPick = numberOfDPNs - result.size();
+            for (int i = 0; i < nbrOfProhibitedDpnsToPick; i++) {
+                result.add(dpnIdPool.get(i));
+            }
+        }
+        return result;
+    }
+
+    public static void updateVpnInterface(DataBroker broker,String interfaceName, BigInteger dpnId, String vpnInstanceName, Boolean isScheduledToRemove){
+        InstanceIdentifier<VpnInterface> interfaceId = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
+        VpnInterface interfaceToUpdate = new VpnInterfaceBuilder().setKey(new VpnInterfaceKey(interfaceName)).setName(interfaceName)
+                .setDpnId(dpnId).setVpnInstanceName(vpnInstanceName).setScheduledForRemove(isScheduledToRemove).build();
+        VpnUtil.syncUpdate(broker, LogicalDatastoreType.OPERATIONAL, interfaceId, interfaceToUpdate);
+    }
+
+    protected static void createVpnPortFixedIpToPort(DataBroker broker, String vpnName, String fixedIp,
+                                                     String portName, String macAddress, boolean isSubnetIp, boolean isConfig,
+                                                     boolean isLearnt) {
+        InstanceIdentifier<VpnPortipToPort> id = buildVpnPortipToPortIdentifier(vpnName, fixedIp);
+        VpnPortipToPortBuilder builder = new VpnPortipToPortBuilder().setKey(
+                new VpnPortipToPortKey(fixedIp, vpnName)).setVpnName(vpnName).setPortFixedip(fixedIp).setPortName(portName)
+                .setMacAddress(macAddress).setSubnetIp(isSubnetIp).setConfig(isConfig).setLearnt(isLearnt);
+        MDSALUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION, id, builder.build());
+        LOG.debug("ARP learned for fixedIp: {}, vpn {}, interface {}, mac {}, isSubnetIp {} added to VpnPortipToPort DS",
+                fixedIp, vpnName, portName, macAddress, isLearnt);
+    }
+
+    protected static void updateVpnPortFixedIpToPort(DataBroker broker, String vpnName, String fixedIp,
+                                                     String portName, String macAddress, boolean isSubnetIp,boolean isConfig,
+                                                     boolean isLearnt) {
+        InstanceIdentifier<VpnPortipToPort> id = buildVpnPortipToPortIdentifier(vpnName, fixedIp);
+        VpnPortipToPortBuilder builder = new VpnPortipToPortBuilder().setKey(
+                new VpnPortipToPortKey(fixedIp, vpnName)).setVpnName(vpnName).setPortFixedip(fixedIp).setPortName(portName)
+                .setMacAddress(macAddress).setSubnetIp(isSubnetIp).setConfig(isConfig).setLearnt(isLearnt);;
+        MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, id, builder.build());
+        LOG.debug("Updated Arp learnt fixedIp: {}, vpn {}, interface {}, mac {}, isLearnt {} Updated to VpnPortipToPort DS",
+                fixedIp, vpnName, portName, macAddress, isLearnt);
+    }
+
+    protected static void removeVpnPortFixedIpToPort(DataBroker broker, String vpnName, String fixedIp) {
+        InstanceIdentifier<VpnPortipToPort> id = buildVpnPortipToPortIdentifier(vpnName, fixedIp);
+        MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, id);
+        LOG.debug("Delete learned ARP for fixedIp: {}, vpn {} removed from VpnPortipToPort DS",
+                fixedIp, vpnName);
+    }
+
+    static InstanceIdentifier<VpnPortipToPort> buildVpnPortipToPortIdentifier(String vpnName, String fixedIp) {
+        InstanceIdentifier<VpnPortipToPort> id = InstanceIdentifier.builder(NeutronVpnPortipPortData.class).child
+                (VpnPortipToPort.class, new VpnPortipToPortKey(fixedIp, vpnName)).build();
+        return id;
+    }
+
+    static VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
+        InstanceIdentifier id = buildVpnPortipToPortIdentifier(vpnName, fixedIp);
+        Optional<VpnPortipToPort> vpnPortipToPortData = read(broker, LogicalDatastoreType.CONFIGURATION, id);
+        if (vpnPortipToPortData.isPresent()) {
+            return (vpnPortipToPortData.get());
+        }
+        return null;
+    }
 }
\ No newline at end of file
index 2af58d78fc047af4f48e6d6bf5f5905154a20e2d..38642878c9a74b918425ad56dac206785d107fd0 100644 (file)
@@ -12,12 +12,20 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
 import org.opendaylight.controller.md.sal.binding.api.NotificationService;
+import org.opendaylight.controller.sal.binding.api.BindingAwareBroker;
 import org.opendaylight.controller.sal.binding.api.BindingAwareBroker.ProviderContext;
 import org.opendaylight.controller.sal.binding.api.BindingAwareProvider;
+import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry;
 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
+import org.opendaylight.netvirt.vpnmanager.intervpnlink.InterVpnLinkListener;
+import org.opendaylight.netvirt.vpnmanager.intervpnlink.InterVpnLinkNodeListener;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingService;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
@@ -26,7 +34,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 import org.opendaylight.yangtools.yang.common.RpcResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,40 +43,58 @@ public class VpnserviceProvider implements BindingAwareProvider, IVpnManager, Au
     private static final Logger LOG = LoggerFactory.getLogger(VpnserviceProvider.class);
     private VpnInterfaceManager vpnInterfaceManager;
     private VpnManager vpnManager;
+    private ArpScheduler arpscheduler;
     private IBgpManager bgpManager;
+    private FibRpcService fibService;
     private IFibManager fibManager;
     private IMdsalApiManager mdsalManager;
-    private OdlInterfaceRpcService interfaceManager;
+    private OdlInterfaceRpcService odlInterfaceRpcService;
     private ItmRpcService itmProvider;
     private IdManagerService idManager;
     private OdlArputilService arpManager;
     private NeutronvpnService neuService;
+    private NotificationService notificationService;
+    private RpcProviderRegistry rpcProviderRegistry;
+    private BindingAwareBroker.RpcRegistration<VpnRpcService> rpcRegistration;
+    private NotificationPublishService notificationPublishService;
     private PacketProcessingService m_packetProcessingService;
     private SubnetRoutePacketInHandler subnetRoutePacketInHandler;
-    private NotificationService notificationService;
+    private InterVpnLinkListener interVpnLinkListener;
+    private DataBroker dataBroker;
+    private InterVpnLinkNodeListener interVpnLinkNodeListener;
+
 
     @Override
     public void onSessionInitiated(ProviderContext session) {
         LOG.info("VpnserviceProvider Session Initiated");
         try {
-            final  DataBroker dataBroker = session.getSALService(DataBroker.class);
+            dataBroker = session.getSALService(DataBroker.class);
             vpnManager = new VpnManager(dataBroker, bgpManager);
             vpnManager.setIdManager(idManager);
             vpnInterfaceManager = new VpnInterfaceManager(dataBroker, bgpManager, notificationService);
             vpnInterfaceManager.setMdsalManager(mdsalManager);
-            vpnInterfaceManager.setInterfaceManager(interfaceManager);
+            vpnInterfaceManager.setIfaceMgrRpcService(odlInterfaceRpcService);
             vpnInterfaceManager.setITMProvider(itmProvider);
             vpnInterfaceManager.setIdManager(idManager);
             vpnInterfaceManager.setArpManager(arpManager);
             vpnInterfaceManager.setNeutronvpnManager(neuService);
+            vpnInterfaceManager.setNotificationPublishService(notificationPublishService);
+            vpnManager.setVpnInterfaceManager(vpnInterfaceManager);
+            fibService = rpcProviderRegistry.getRpcService(FibRpcService.class);
+            vpnInterfaceManager.setFibRpcService(fibService);
+            VpnRpcService vpnRpcService = new VpnRpcServiceImpl(idManager, vpnInterfaceManager, dataBroker);
+            rpcRegistration = getRpcProviderRegistry().addRpcImplementation(VpnRpcService.class, vpnRpcService);
             //Handles subnet route entries
             subnetRoutePacketInHandler = new SubnetRoutePacketInHandler(dataBroker, idManager);
             m_packetProcessingService = session.getRpcService(PacketProcessingService.class);
             subnetRoutePacketInHandler.setPacketProcessingService(m_packetProcessingService);
             notificationService.registerNotificationListener(subnetRoutePacketInHandler);
-            vpnManager.setVpnInterfaceManager(vpnInterfaceManager);
+            interVpnLinkListener = new InterVpnLinkListener(dataBroker, idManager, mdsalManager, bgpManager,
+                                                            notificationPublishService);
+            interVpnLinkNodeListener = new InterVpnLinkNodeListener(dataBroker, mdsalManager);
             createIdPool();
             RouterInterfaceListener routerListener = new RouterInterfaceListener(dataBroker);
+            arpscheduler = new ArpScheduler(odlInterfaceRpcService, dataBroker);
             routerListener.setVpnInterfaceManager(vpnInterfaceManager);
         } catch (Exception e) {
             LOG.error("Error initializing services", e);
@@ -89,8 +114,8 @@ public class VpnserviceProvider implements BindingAwareProvider, IVpnManager, Au
         this.mdsalManager = mdsalManager;
     }
 
-    public void setInterfaceManager(OdlInterfaceRpcService interfaceManager) {
-        this.interfaceManager = interfaceManager;
+    public void setOdlInterfaceRpcService(OdlInterfaceRpcService odlInterfaceRpcService) {
+        this.odlInterfaceRpcService = odlInterfaceRpcService;
     }
 
     public void setITMProvider(ItmRpcService itmProvider) {
@@ -105,6 +130,18 @@ public class VpnserviceProvider implements BindingAwareProvider, IVpnManager, Au
         this.arpManager = arpManager;
     }
 
+    public void setRpcProviderRegistry(RpcProviderRegistry rpcProviderRegistry) {
+        this.rpcProviderRegistry = rpcProviderRegistry;
+    }
+
+    private RpcProviderRegistry getRpcProviderRegistry() {
+        return rpcProviderRegistry;
+    }
+
+    public void setNotificationPublishService(NotificationPublishService notificationPublishService) {
+        this.notificationPublishService = notificationPublishService;
+    }
+
     private void createIdPool() {
         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
@@ -125,6 +162,8 @@ public class VpnserviceProvider implements BindingAwareProvider, IVpnManager, Au
     public void close() throws Exception {
         vpnManager.close();
         vpnInterfaceManager.close();
+        interVpnLinkListener.close();
+        interVpnLinkNodeListener.close();
 
     }
 
@@ -137,14 +176,14 @@ public class VpnserviceProvider implements BindingAwareProvider, IVpnManager, Au
 
     @Override
     public void addExtraRoute(String destination, String nextHop, String rd, String routerID, int label) {
-        LOG.info("Adding extra route with destination {} and nexthop {}", destination, nextHop);
-        vpnInterfaceManager.addExtraRoute(destination, nextHop, rd, routerID, label, null);
+        LOG.info("Adding extra route with destination {}, nextHop {} and label{}", destination, nextHop, label);
+        vpnInterfaceManager.addExtraRoute(destination, nextHop, rd, routerID, label, /*intfName*/ null);
     }
 
     @Override
-    public void delExtraRoute(String destination, String rd, String routerID) {
-        LOG.info("Deleting extra route with destination {}", destination);
-        vpnInterfaceManager.delExtraRoute(destination, rd, routerID);
+    public void delExtraRoute(String destination, String nextHop, String rd, String routerID) {
+        LOG.info("Deleting extra route with destination {} and nextHop {}", destination, nextHop);
+        vpnInterfaceManager.delExtraRoute(destination, nextHop, rd, routerID, null);
     }
 
     @Override
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkListener.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkListener.java
new file mode 100644 (file)
index 0000000..a414805
--- /dev/null
@@ -0,0 +1,456 @@
+/*
+ * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager.intervpnlink;
+
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
+import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
+import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
+import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.genius.interfacemanager.globals.IfmConstants;
+import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
+import org.opendaylight.genius.mdsalutil.MDSALUtil;
+import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
+import org.opendaylight.netvirt.bgpmanager.api.RouteOrigin;
+import org.opendaylight.netvirt.vpnmanager.VpnConstants;
+import org.opendaylight.netvirt.vpnmanager.VpnUtil;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.*;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinkCreationError;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinkCreationErrorBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinks;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.creation.error.InterVpnLinkCreationErrorMessage;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.creation.error.InterVpnLinkCreationErrorMessageBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLinkKey;
+import org.opendaylight.yangtools.concepts.ListenerRegistration;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.opendaylight.yangtools.yang.common.RpcResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+public class InterVpnLinkListener extends AbstractDataChangeListener<InterVpnLink> {
+    private static final Logger logger = LoggerFactory.getLogger(InterVpnLinkListener.class);
+    private static final String NBR_OF_DPNS_PROPERTY_NAME = "vpnservice.intervpnlink.number.dpns";
+    private static final int INVALID_ID = 0;
+
+    private IMdsalApiManager mdsalManager;
+    private IdManagerService idManager;
+    private IBgpManager bgpManager;
+    private NotificationPublishService notificationsService;
+
+    private ListenerRegistration<DataChangeListener> listenerRegistration;
+    private final DataBroker broker;
+
+
+    public InterVpnLinkListener(DataBroker db, IdManagerService idManager, IMdsalApiManager mdsalManager,
+                                IBgpManager bgpManager, NotificationPublishService notifService) {
+        super(InterVpnLink.class);
+        this.broker = db;
+        this.idManager = idManager;
+        this.mdsalManager = mdsalManager;
+        this.bgpManager = bgpManager;
+        this.notificationsService = notifService;
+        this.registerListener(db);
+    }
+
+    public void close() {
+        if (listenerRegistration != null) {
+            try {
+                listenerRegistration.close();
+            } catch (final Exception e) {
+                logger.error("Error when cleaning up DataChangeListener.", e);
+            }
+            listenerRegistration = null;
+        }
+    }
+
+
+    /**
+     * Register the databroker to all the possible changes from MDSAL - InterVpnLink class
+     *
+     * @param db dataBroker service reference
+     */
+    private void registerListener(DataBroker db) {
+        try {
+            this.listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
+                                                                      this.getWildCardPath(),
+                                                                      this,
+                                                                      AsyncDataBroker.DataChangeScope.SUBTREE);
+        } catch (Exception e) {
+            logger.error("InterVpnLinkListener: DataChange listener registration fail!", e);
+            throw new IllegalStateException("InterVpnLinkListener: registration Listener failed.", e);
+        }
+    }
+
+    private InstanceIdentifier<InterVpnLink> getWildCardPath() {
+        return InstanceIdentifier.create(InterVpnLinks.class).child(InterVpnLink.class);
+    }
+
+    private String getInterVpnLinkIfaceName(String vpnUuid, BigInteger dpnId ) {
+        return String.format("InterVpnLink.%s.%s", vpnUuid, dpnId.toString());
+    }
+
+    @Override
+    protected void add(InstanceIdentifier<InterVpnLink> identifier, InterVpnLink add) {
+
+        int numberOfDpns = Integer.getInteger(NBR_OF_DPNS_PROPERTY_NAME, 1);
+        // Create VpnLink state
+        InstanceIdentifier<InterVpnLinkState> vpnLinkStateIid = VpnUtil.getInterVpnLinkStateIid(add.getName());
+        InterVpnLinkState vpnLinkState = new InterVpnLinkStateBuilder().setInterVpnLinkName(add.getName()).build();
+        MDSALUtil.syncWrite(broker, LogicalDatastoreType.CONFIGURATION, vpnLinkStateIid, vpnLinkState);
+
+        InterVpnLinkKey key = add.getKey();
+        Uuid firstEndpointVpnUuid = add.getFirstEndpoint().getVpnUuid();
+        Uuid secondEndpointVpnUuid = add.getSecondEndpoint().getVpnUuid();
+        // First VPN
+        if (!checkVpnAvailability(key, firstEndpointVpnUuid)) {
+            String errMsg = String.format("Vpn already associated with a previous inter-vpn-link {}",
+                                          firstEndpointVpnUuid);
+            logger.error(errMsg);
+            setInError(vpnLinkStateIid, vpnLinkState, errMsg);
+            return;
+        }
+
+        // Second VPN
+        if (!checkVpnAvailability(key, secondEndpointVpnUuid)) {
+            String errMsg = String.format("Vpn already associated with a previous inter-vpn-link {}",
+                                          secondEndpointVpnUuid);
+            logger.error(errMsg);
+            setInError(vpnLinkStateIid, vpnLinkState, errMsg);
+            return;
+        }
+
+        // TODO: Doing like this we are retrieving operative DPNs from MDSAL when we just need one. Fix it
+        List<BigInteger> firstDpnList = VpnUtil.pickRandomDPNs(broker, numberOfDpns, null);
+        if (firstDpnList != null && !firstDpnList.isEmpty()) {
+            // TODO: Limitation to be solved later
+            // List<BigInteger> secondDpnList = VpnUtil.pickRandomDPNs(broker, numberOfDpns, firstDpnList);
+            List<BigInteger> secondDpnList = firstDpnList;
+
+            Integer firstVpnLportTag = allocateVpnLinkLportTag(key.getName() + firstEndpointVpnUuid.getValue());
+            Integer secondVpnLportTag = allocateVpnLinkLportTag(key.getName() + secondEndpointVpnUuid.getValue());
+            FirstEndpointState firstEndPointState =
+                new FirstEndpointStateBuilder().setVpnUuid(firstEndpointVpnUuid).setDpId(firstDpnList)
+                                               .setLportTag(firstVpnLportTag).build();
+            SecondEndpointState secondEndPointState =
+                new SecondEndpointStateBuilder().setVpnUuid(secondEndpointVpnUuid).setDpId(secondDpnList)
+                                                .setLportTag(secondVpnLportTag).build();
+
+            InterVpnLinkUtil.updateInterVpnLinkState(broker, add.getName(), InterVpnLinkState.State.Active, firstEndPointState,
+                                            secondEndPointState);
+
+            // Note that in the DPN of the firstEndpoint we install the lportTag of the secondEndpoint and viceversa
+            InterVpnLinkUtil.installLPortDispatcherTableFlow(broker, mdsalManager, add, firstDpnList,
+                                                    secondEndpointVpnUuid, secondVpnLportTag);
+            InterVpnLinkUtil.installLPortDispatcherTableFlow(broker, mdsalManager, add, secondDpnList,
+                                                    firstEndpointVpnUuid, firstVpnLportTag);
+            // Update the VPN -> DPNs Map.
+            // Note: when a set of DPNs is calculated for Vpn1, these DPNs are added to the VpnToDpn map of Vpn2. Why?
+            // because we do the handover from Vpn1 to Vpn2 in those DPNs, so in those DPNs we must know how to reach
+            // to Vpn2 targets. If new Vpn2 targets are added later, the Fib will be maintained in these DPNs even if
+            // Vpn2 is not physically present there.
+            InterVpnLinkUtil.updateVpnToDpnMap(broker, firstDpnList, secondEndpointVpnUuid);
+            InterVpnLinkUtil.updateVpnToDpnMap(broker, secondDpnList, firstEndpointVpnUuid);
+
+            // Now, if the corresponding flags are activated, there will be some routes exchange
+            leakRoutesIfNeeded(add);
+        } else {
+            // If there is no connection to DPNs, the InterVpnLink is created and the InterVpnLinkState is also created
+            // with the corresponding LPortTags but no DPN is assigned since there is no DPN operative.
+            Integer firstVpnLportTag = allocateVpnLinkLportTag(key.getName() + firstEndpointVpnUuid.getValue());
+            Integer secondVpnLportTag = allocateVpnLinkLportTag(key.getName() + secondEndpointVpnUuid.getValue());
+            FirstEndpointState firstEndPointState =
+                new FirstEndpointStateBuilder().setVpnUuid(firstEndpointVpnUuid)
+                                               .setLportTag(firstVpnLportTag).build();
+            SecondEndpointState secondEndPointState =
+                new SecondEndpointStateBuilder().setVpnUuid(secondEndpointVpnUuid)
+                                                .setLportTag(secondVpnLportTag).build();
+            InterVpnLinkUtil.updateInterVpnLinkState(broker, add.getName(), InterVpnLinkState.State.Error, firstEndPointState,
+                                            secondEndPointState);
+        }
+
+
+    }
+
+    private void leakRoutesIfNeeded(InterVpnLink vpnLink) {
+
+        // The type of routes to exchange depend on the leaking flags that have been activated
+        List<RouteOrigin> originsToConsider = new ArrayList<>();
+        if ( vpnLink.isBgpRoutesLeaking() ) {
+            originsToConsider.add(RouteOrigin.BGP);
+        }
+
+        /* For now, only BGP leaking. Leave this here for when the other leakings are activated
+        if ( vpnLink.isConnectedRoutesLeaking() ) {
+            originsToConsider.add(RouteOrigin.CONNECTED);
+        }
+        if ( vpnLink.isStaticRoutesLeaking() ) {
+            originsToConsider.add(RouteOrigin.STATIC);
+            NOTE: There are 2 types of static routes depending on the nexthop:
+              + static route when nexthop is a VM, the Dc-GW or a DPNIP
+              + static route when nexthop is an InterVPN Link
+            Only the 1st type should be considered since the 2nd has a special treatment
+        } */
+        String vpn1Uuid = vpnLink.getFirstEndpoint().getVpnUuid().getValue();
+        String vpn2Uuid = vpnLink.getSecondEndpoint().getVpnUuid().getValue();
+
+        if ( ! originsToConsider.isEmpty() ) {
+            // 1st Endpoint ==> 2nd endpoint
+            leakRoutes(vpnLink, vpn1Uuid, vpn2Uuid, originsToConsider);
+
+            // 2nd Endpoint ==> 1st endpoint
+            leakRoutes(vpnLink, vpnLink.getSecondEndpoint().getVpnUuid().getValue(),
+                       vpnLink.getFirstEndpoint().getVpnUuid().getValue(),
+                       originsToConsider);
+        }
+
+        // Static routes in Vpn1 pointing to Vpn2's endpoint
+        leakExtraRoutesToVpnEndpoint(vpnLink, vpn1Uuid, vpn2Uuid);
+
+        // Static routes in Vpn2 pointing to Vpn1's endpoint
+        leakExtraRoutesToVpnEndpoint(vpnLink, vpn2Uuid, vpn1Uuid);
+    }
+
+    private void leakRoutes(InterVpnLink vpnLink, String srcVpnUuid, String dstVpnUuid,
+                            List<RouteOrigin> originsToConsider) {
+        String srcVpnRd = VpnUtil.getVpnRd(broker, srcVpnUuid);
+        String dstVpnRd = VpnUtil.getVpnRd(broker, dstVpnUuid);
+        List<VrfEntry> srcVpnRemoteVrfEntries = VpnUtil.getVrfEntriesByOrigin(broker, srcVpnRd, originsToConsider);
+        for ( VrfEntry vrfEntry : srcVpnRemoteVrfEntries ) {
+            long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                                             VpnUtil.getNextHopLabelKey(dstVpnRd, vrfEntry.getDestPrefix()));
+
+            VpnUtil.leakRoute(broker, bgpManager, vpnLink, srcVpnUuid, dstVpnUuid,
+                              vrfEntry.getDestPrefix(), label);
+        }
+    }
+
+    /*
+     * Checks if there are static routes in Vpn1 whose nexthop is Vpn2's endpoint. Those routes must be leaked to Vpn1.
+     *
+     * @param vpnLink
+     * @param vpn1Uuid
+     * @param vpn2Uuid
+     */
+    private void leakExtraRoutesToVpnEndpoint(InterVpnLink vpnLink, String vpn1Uuid, String vpn2Uuid) {
+
+        String vpn1Rd = VpnUtil.getVpnRd(broker, vpn1Uuid);
+        String vpn2Endpoint = vpnLink.getSecondEndpoint().getIpAddress().getValue();
+        List<VrfEntry> allVpnVrfEntries = VpnUtil.getAllVrfEntries(broker, vpn1Rd);
+        for ( VrfEntry vrfEntry : allVpnVrfEntries ) {
+            if ( vrfEntry.getNextHopAddressList() != null
+                && vrfEntry.getNextHopAddressList().contains(vpn2Endpoint) ) {
+                // Vpn1 has a route pointing to Vpn2's endpoint. Forcing the leaking of the route will update the
+                // BGP accordingly
+                long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
+                                                  VpnUtil.getNextHopLabelKey(vpn1Rd, vrfEntry.getDestPrefix()));
+
+                VpnUtil.leakRoute(broker, bgpManager, vpnLink, vpn2Uuid, vpn1Uuid, vrfEntry.getDestPrefix(),
+                                  label, RouteOrigin.value(vrfEntry.getOrigin()));
+            }
+        }
+
+    }
+
+    private boolean checkVpnAvailability(InterVpnLinkKey key, Uuid vpnId) {
+        Preconditions.checkNotNull(vpnId);
+
+        List<InterVpnLink> interVpnLinks = VpnUtil.getAllInterVpnLinks(broker);
+        if ( interVpnLinks != null ) {
+            for (InterVpnLink interVpnLink : interVpnLinks) {
+                if (!key.equals(interVpnLink.getKey())
+                    && (vpnId.equals(interVpnLink.getFirstEndpoint().getVpnUuid())
+                        || vpnId.equals(interVpnLink.getSecondEndpoint().getVpnUuid()))) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+
+    @Override
+    protected void remove(InstanceIdentifier<InterVpnLink> identifier, InterVpnLink del) {
+
+        // Remove learnt routes
+        // Remove entries in the LPortDispatcher table
+        // Remove the corresponding entries in InterVpnLinkState
+
+        // For each endpoint, remove all routes that have been learnt by intervpnLink
+        String vpn1Uuid = del.getFirstEndpoint().getVpnUuid().getValue();
+        String rd1 = VpnUtil.getVpnRdFromVpnInstanceConfig(broker, vpn1Uuid);
+        VpnUtil.removeVrfEntriesByOrigin(broker, rd1, RouteOrigin.INTERVPN);
+        VpnUtil.removeVrfEntriesByNexthop(broker, rd1, del.getSecondEndpoint().getIpAddress().getValue());
+
+        String vpn2Uuid = del.getSecondEndpoint().getVpnUuid().getValue();
+        String rd2 = VpnUtil.getVpnRdFromVpnInstanceConfig(broker, vpn2Uuid);
+        VpnUtil.removeVrfEntriesByOrigin(broker, rd2, RouteOrigin.INTERVPN);
+        VpnUtil.removeVrfEntriesByNexthop(broker, rd2, del.getFirstEndpoint().getIpAddress().getValue());
+
+        InterVpnLinkState interVpnLinkState = VpnUtil.getInterVpnLinkState(broker, del.getName());
+        Integer firstEndpointLportTag = interVpnLinkState.getFirstEndpointState().getLportTag();
+
+        Integer secondEndpointLportTag = interVpnLinkState.getSecondEndpointState().getLportTag();
+
+        // Remmoving the flow entries in LPortDispatcher table in 1st Endpoint DPNs
+        for ( BigInteger dpnId : interVpnLinkState.getFirstEndpointState().getDpId() ) {
+            String flowRef = InterVpnLinkUtil.getLportDispatcherFlowRef(del.getName(), secondEndpointLportTag);
+            FlowKey flowKey = new FlowKey(new FlowId(flowRef));
+            Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
+                                         .setTableId(NwConstants.LPORT_DISPATCHER_TABLE).setFlowName(flowRef)
+                                         .build();
+            mdsalManager.removeFlow(dpnId, flow);
+
+            // Also remove the 'fake' iface from the VpnToDpn map
+            VpnUtil.removeIfaceFromVpnToDpnMap(broker, rd1, dpnId, getInterVpnLinkIfaceName(vpn1Uuid, dpnId));
+        }
+
+        // Removing the flow entries in 2nd Endpoint DPNs
+        for ( BigInteger dpnId : interVpnLinkState.getSecondEndpointState().getDpId() ) {
+            String flowRef = InterVpnLinkUtil.getLportDispatcherFlowRef(del.getName(), firstEndpointLportTag);
+            FlowKey flowKey = new FlowKey(new FlowId(flowRef));
+            Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
+                                         .setTableId(NwConstants.LPORT_DISPATCHER_TABLE).setFlowName(flowRef)
+                                         .build();
+            mdsalManager.removeFlow(dpnId, flow);
+
+            // Also remove the 'fake' iface from the VpnToDpn map
+            VpnUtil.removeIfaceFromVpnToDpnMap(broker, rd2, dpnId, getInterVpnLinkIfaceName(vpn2Uuid, dpnId));
+        }
+
+        // Release idManager wit LPortTag associated to endpoints
+        InterVpnLinkKey key = del.getKey();
+        Uuid firstEndpointVpnUuid = del.getFirstEndpoint().getVpnUuid();
+        Uuid secondEndpointVpnUuid = del.getSecondEndpoint().getVpnUuid();
+        releaseVpnLinkLPortTag(key.getName() + firstEndpointVpnUuid.getValue());
+        releaseVpnLinkLPortTag(key.getName() + secondEndpointVpnUuid.getValue());
+
+        // Routes with nextHop pointing to an end-point of the inter-vpn-link are populated into FIB table.
+        // The action in that case is a nx_resubmit to LPortDispatcher table. This is done in FibManager.
+        // At this point. we need to check if is there any entry in FIB table pointing to LPortDispatcher table.
+        // Remove it in that case.
+
+        // 1stEndPoint dpns
+        for ( BigInteger dpnId : interVpnLinkState.getFirstEndpointState().getDpId() ) {
+            removeRouteFromInterVpnLink(dpnId, del.getName(), del.getSecondEndpoint().getIpAddress().getValue());
+        }
+
+        // 2ndtEndPoint dpns
+        for ( BigInteger dpnId : interVpnLinkState.getSecondEndpointState().getDpId() ) {
+            removeRouteFromInterVpnLink(dpnId, del.getName(), del.getFirstEndpoint().getIpAddress().getValue());
+        }
+
+        // Removing the InterVpnLinkState
+        InstanceIdentifier<InterVpnLinkState> interVpnLinkStateIid = VpnUtil.getInterVpnLinkStateIid(del.getName());
+        VpnUtil.delete(broker, LogicalDatastoreType.CONFIGURATION, interVpnLinkStateIid);
+    }
+
+    private void releaseVpnLinkLPortTag(String idKey) {
+        ReleaseIdInput releaseIdInput = new ReleaseIdInputBuilder().setPoolName(IfmConstants.IFM_IDPOOL_NAME)
+                                                                   .setIdKey(idKey).build();
+        idManager.releaseId(releaseIdInput);
+    }
+
+    @Override
+    protected void update(InstanceIdentifier<InterVpnLink> identifier, InterVpnLink original, InterVpnLink update) {
+     // TODO
+    }
+
+    private String getInterVpnFibFlowRef(BigInteger dpnId, short tableId, String interVpnLinkName, String nextHop ) {
+        return new StringBuilder(64).append(VpnConstants.FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR)
+            .append(tableId).append(NwConstants.FLOWID_SEPARATOR)
+            .append(interVpnLinkName).append(NwConstants.FLOWID_SEPARATOR)
+            .append(nextHop).toString();
+    }
+
+    private void removeRouteFromInterVpnLink(BigInteger dpnId, String interVpnLinkName, final String nextHop) {
+        String flowRef = getInterVpnFibFlowRef(dpnId, NwConstants.L3_FIB_TABLE, interVpnLinkName, nextHop);
+        FlowKey flowKey = new FlowKey(new FlowId(flowRef));
+        Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef))
+                .setTableId(NwConstants.L3_FIB_TABLE).setFlowName(flowRef)
+                .build();
+        mdsalManager.removeFlow(dpnId, flow);
+    }
+
+
+    private Integer allocateVpnLinkLportTag(String idKey) {
+        AllocateIdInput getIdInput =
+                new AllocateIdInputBuilder().setPoolName(IfmConstants.IFM_IDPOOL_NAME).setIdKey(idKey).build();
+        try {
+            Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
+            RpcResult<AllocateIdOutput> rpcResult = result.get();
+            if(rpcResult.isSuccessful()) {
+                return rpcResult.getResult().getIdValue().intValue();
+            } else {
+                logger.warn("RPC Call to Get Unique Id returned with Errors {}", rpcResult.getErrors());
+            }
+        } catch (InterruptedException | ExecutionException e) {
+            logger.warn("Exception when getting Unique Id",e);
+        }
+        return INVALID_ID;
+    }
+
+    protected void setInError(final InstanceIdentifier<InterVpnLinkState> vpnLinkStateIid,
+                              final InterVpnLinkState vpnLinkState,
+                              String errorMsg) {
+        // Setting InterVPNLink in error state in MDSAL
+        InterVpnLinkState vpnLinkErrorState =
+           new InterVpnLinkStateBuilder(vpnLinkState).setState(InterVpnLinkState.State.Error)
+                                                     .setErrorDescription(errorMsg)
+                                                     .build();
+        WriteTransaction tx = broker.newWriteOnlyTransaction();
+        tx.put(LogicalDatastoreType.CONFIGURATION, vpnLinkStateIid, vpnLinkErrorState, true);
+        tx.submit();
+
+        // Sending out an error Notification
+        InterVpnLinkCreationErrorMessage errMsg =
+            new InterVpnLinkCreationErrorMessageBuilder().setErrorMessage(errorMsg).build();
+        InterVpnLinkCreationError notif =
+            new InterVpnLinkCreationErrorBuilder().setInterVpnLinkCreationErrorMessage(errMsg).build();
+        final ListenableFuture<? extends Object> eventFuture = this.notificationsService.offerNotification(notif);
+        Futures.addCallback(eventFuture, new FutureCallback<Object>() {
+            @Override
+            public void onFailure(Throwable error) {
+                logger.warn("Error when sending notification about InterVpnLink creation issue. InterVpnLink name={}. Error={}",
+                            vpnLinkState.getInterVpnLinkName(), vpnLinkState, error);
+            }
+
+            @Override
+            public void onSuccess(Object arg) {
+                logger.trace("Error notification for InterVpnLink successfully sent. VpnLink={} error={}",
+                             vpnLinkState.getInterVpnLinkName(), vpnLinkState);
+            }
+        });
+    }
+}
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkNodeAddTask.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkNodeAddTask.java
new file mode 100644 (file)
index 0000000..ed8edd0
--- /dev/null
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager.intervpnlink;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
+import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.netvirt.vpnmanager.VpnUtil;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+public class InterVpnLinkNodeAddTask implements Callable<List<ListenableFuture<Void>>> {
+    private static final String NBR_OF_DPNS_PROPERTY_NAME = "vpnservice.intervpnlink.number.dpns";
+
+    private DataBroker broker;
+    private BigInteger dpnId;
+    final IMdsalApiManager mdsalManager;
+
+    public InterVpnLinkNodeAddTask(final DataBroker broker, final IMdsalApiManager mdsalMgr, final BigInteger dpnId) {
+        this.broker = broker;
+        this.dpnId = dpnId;
+        this.mdsalManager = mdsalMgr;
+    }
+
+    @Override
+    public List<ListenableFuture<Void>> call() throws Exception {
+        List<ListenableFuture<Void>> result = new ArrayList<ListenableFuture<Void>>();
+        // check if there is any inter-vpn-link in with erroneous state
+        List<InterVpnLinkState> allInterVpnLinkState = InterVpnLinkUtil.getAllInterVpnLinkState(broker);
+        int numberOfDpns = Integer.getInteger(NBR_OF_DPNS_PROPERTY_NAME, 1);
+
+        List<BigInteger> firstDpnList = new ArrayList<BigInteger>();
+        List<BigInteger> secondDpnList = new ArrayList<BigInteger>();
+        for (InterVpnLinkState interVpnLinkState : allInterVpnLinkState) {
+            // if the inter-vpn-link is erroneous and any of its endPoints has no dpns associated
+            if (shouldConfigureLinkIntoDpn(interVpnLinkState, this.dpnId, numberOfDpns)) {
+                firstDpnList.add(dpnId);
+                secondDpnList = firstDpnList;
+                // TODO: Limitation to be solved later
+                // List<BigInteger> secondDpnList = VpnUtil.pickRandomDPNs(broker, numberOfDpns, firstDpnList);
+                secondDpnList = firstDpnList;
+                installLPortDispatcherTable(interVpnLinkState, firstDpnList, secondDpnList);
+                CheckedFuture<Void, TransactionCommitFailedException> futures =
+                        updateInterVpnLinkState(interVpnLinkState, firstDpnList, secondDpnList, numberOfDpns);
+                result.add(futures);
+            }
+        }
+        return result;
+    }
+
+    private boolean shouldConfigureLinkIntoDpn(InterVpnLinkState interVpnLinkState, BigInteger dpnId, int numberOfDpns) {
+
+        if (interVpnLinkState.getState().equals(InterVpnLinkState.State.Error)) {
+            if ((interVpnLinkState.getFirstEndpointState().getDpId() == null
+               || interVpnLinkState.getFirstEndpointState().getDpId().isEmpty())
+               || (interVpnLinkState.getSecondEndpointState().getDpId() == null
+               || interVpnLinkState.getSecondEndpointState().getDpId().isEmpty())) {
+                return true;
+            } else if (!interVpnLinkState.getFirstEndpointState().getDpId().contains(dpnId)
+                    && !interVpnLinkState.getSecondEndpointState().getDpId().contains(dpnId)
+                    && (interVpnLinkState.getFirstEndpointState().getDpId().size() < numberOfDpns)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private CheckedFuture<Void, TransactionCommitFailedException>
+                updateInterVpnLinkState(InterVpnLinkState interVpnLinkState, List<BigInteger> firstDpnList,
+                                        List<BigInteger> secondDpnList, int numberOfDpns) {
+        FirstEndpointState firstEndPointState =
+                new FirstEndpointStateBuilder(interVpnLinkState.getFirstEndpointState())
+                                                               .setDpId(firstDpnList).build();
+        SecondEndpointState secondEndPointState =
+                new SecondEndpointStateBuilder(interVpnLinkState.getSecondEndpointState())
+                                                                .setDpId(secondDpnList).build();
+        InterVpnLinkState newInterVpnLinkState = new InterVpnLinkStateBuilder(interVpnLinkState)
+                                                                .setState(InterVpnLinkState.State.Active)
+                                                                .setFirstEndpointState(firstEndPointState)
+                                                                .setSecondEndpointState(secondEndPointState)
+                                                                .build();
+        WriteTransaction tx = broker.newWriteOnlyTransaction();
+        tx.merge(LogicalDatastoreType.CONFIGURATION,
+                 VpnUtil.getInterVpnLinkStateIid(interVpnLinkState.getInterVpnLinkName()), newInterVpnLinkState, true);
+        CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
+        return futures;
+    }
+
+    private void installLPortDispatcherTable(InterVpnLinkState interVpnLinkState, List<BigInteger> firstDpnList,
+                                             List<BigInteger> secondDpnList) {
+        Optional<InterVpnLink> vpnLink = InterVpnLinkUtil.getInterVpnLinkByName(broker, interVpnLinkState.getKey().getInterVpnLinkName());
+        if (vpnLink.isPresent()) {
+            Uuid firstEndpointVpnUuid = vpnLink.get().getFirstEndpoint().getVpnUuid();
+            Uuid secondEndpointVpnUuid = vpnLink.get().getSecondEndpoint().getVpnUuid();
+            // Note that in the DPN of the firstEndpoint we install the lportTag of the secondEndpoint and viceversa
+            InterVpnLinkUtil.installLPortDispatcherTableFlow(broker, mdsalManager, vpnLink.get(), firstDpnList,
+                                                    secondEndpointVpnUuid,
+                                                    interVpnLinkState.getSecondEndpointState().getLportTag());
+            InterVpnLinkUtil.installLPortDispatcherTableFlow(broker, mdsalManager, vpnLink.get(), secondDpnList,
+                                                    firstEndpointVpnUuid,
+                                                    interVpnLinkState.getFirstEndpointState().getLportTag());
+            // Update the VPN -> DPNs Map.
+            // Note: when a set of DPNs is calculated for Vpn1, these DPNs are added to the VpnToDpn map of Vpn2. Why?
+            // because we do the handover from Vpn1 to Vpn2 in those DPNs, so in those DPNs we must know how to reach
+            // to Vpn2 targets. If new Vpn2 targets are added later, the Fib will be maintained in these DPNs even if
+            // Vpn2 is not physically present there.
+            InterVpnLinkUtil.updateVpnToDpnMap(broker, firstDpnList, secondEndpointVpnUuid);
+            InterVpnLinkUtil.updateVpnToDpnMap(broker, secondDpnList, firstEndpointVpnUuid);
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkNodeListener.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkNodeListener.java
new file mode 100644 (file)
index 0000000..3fa1360
--- /dev/null
@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager.intervpnlink;
+
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
+import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
+import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
+import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
+import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener;
+import org.opendaylight.genius.mdsalutil.MDSALUtil;
+import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.netvirt.vpnmanager.VpnUtil;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinks;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLinkBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLinkKey;
+import org.opendaylight.yangtools.concepts.ListenerRegistration;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+/**
+ * Listens for Nodes going down, in order to check if the InterVpnLink must be
+ * moved to some other DPN
+ *
+ */
+public class InterVpnLinkNodeListener extends AbstractDataChangeListener<Node>
+                                      implements AutoCloseable {
+
+    private static final Logger logger = LoggerFactory.getLogger(InterVpnLinkNodeListener.class);
+    private ListenerRegistration<DataChangeListener> listenerRegistration;
+    private static final String NBR_OF_DPNS_PROPERTY_NAME = "vpnservice.intervpnlink.number.dpns";
+    private final DataBroker broker;
+    final IMdsalApiManager mdsalManager;
+
+    public InterVpnLinkNodeListener(final DataBroker db, IMdsalApiManager mdsalMgr) {
+        super(Node.class);
+        broker = db;
+        mdsalManager = mdsalMgr;
+        registerListener(db);
+    }
+
+    @Override
+    public void close() {
+        if (listenerRegistration != null) {
+            try {
+                listenerRegistration.close();
+            } catch (final Exception e) {
+                logger.error("Error when cleaning up DataChangeListener.", e);
+            }
+            listenerRegistration = null;
+        }
+    }
+
+    private void registerListener(final DataBroker db) {
+        try {
+            listenerRegistration = db.registerDataChangeListener(LogicalDatastoreType.OPERATIONAL,
+                                                                 getWildCardPath(),
+                                                                 InterVpnLinkNodeListener.this,
+                                                                 AsyncDataBroker.DataChangeScope.SUBTREE);
+        } catch (final Exception e) {
+            logger.error("InterVpnLinkNodeListener: DataChange listener registration fail!", e);
+            throw new IllegalStateException("InterVpnLinkNodeListener: registration Listener failed.", e);
+        }
+    }
+
+    protected InstanceIdentifier<Node> getWildCardPath() {
+        return InstanceIdentifier.create(Nodes.class).child(Node.class);
+    }
+
+    @Override
+    protected void add(InstanceIdentifier<Node> identifier, Node add) {
+        NodeId nodeId = add.getId();
+        String[] node =  nodeId.getValue().split(":");
+        if(node.length < 2) {
+            logger.warn("Unexpected nodeId {}", nodeId.getValue());
+            return;
+        }
+        BigInteger dpId = new BigInteger(node[1]);
+        DataStoreJobCoordinator coordinator = DataStoreJobCoordinator.getInstance();
+        coordinator.enqueueJob("IVpnLink" + dpId.toString(), new InterVpnLinkNodeAddTask(broker, mdsalManager, dpId));
+    }
+
+    @Override
+    protected void remove(InstanceIdentifier<Node> identifier, Node del) {
+        logger.info("Node {} has been deleted", identifier.firstKeyOf(Node.class).toString());
+        NodeId nodeId = del.getId();
+        String[] node =  nodeId.getValue().split(":");
+        if(node.length < 2) {
+            logger.warn("Unexpected nodeId {}", nodeId.getValue());
+            return;
+        }
+        BigInteger dpId = new BigInteger(node[1]);
+        DataStoreJobCoordinator coordinator = DataStoreJobCoordinator.getInstance();
+        coordinator.enqueueJob("IVpnLink" + dpId.toString(), new InterVpnLinkNodeWorker(broker, dpId));
+
+    }
+
+    @Override
+    protected void update(InstanceIdentifier<Node> identifier, Node original, Node update) {
+        logger.info("Node {} has changed", identifier.firstKeyOf(Node.class).toString());
+    }
+
+    protected class InterVpnLinkNodeWorker implements Callable<List<ListenableFuture<Void>>> {
+
+        private DataBroker broker;
+        private BigInteger dpnId;
+
+        public InterVpnLinkNodeWorker(final DataBroker broker, final BigInteger dpnId) {
+            this.broker = broker;
+            this.dpnId = dpnId;
+        }
+        @Override
+        public List<ListenableFuture<Void>> call() throws Exception {
+            List<ListenableFuture<Void>> result = new ArrayList<ListenableFuture<Void>>();
+
+            List<InterVpnLink> allInterVpnLinks = VpnUtil.getAllInterVpnLinks(broker);
+            for ( InterVpnLink interVpnLink : allInterVpnLinks ) {
+                InterVpnLinkState interVpnLinkState = VpnUtil.getInterVpnLinkState(broker, interVpnLink.getName());
+                if ( interVpnLinkState == null ) {
+                    logger.warn("Could not find State info for InterVpnLink={}", interVpnLink.getName());
+                    continue;
+                }
+
+                if ( interVpnLinkState.getFirstEndpointState().getDpId().contains(dpnId)
+                     || interVpnLinkState.getSecondEndpointState().getDpId().contains(dpnId) ) {
+                    // InterVpnLink affected by Node DOWN.
+                    // Lets move the InterVpnLink to some other place. Basically, remove it and create it again
+                    InstanceIdentifier<InterVpnLink> interVpnLinkIid =
+                        InstanceIdentifier.builder(InterVpnLinks.class)
+                                          .child(InterVpnLink.class, new InterVpnLinkKey(interVpnLink.getName()))
+                                          .build();
+                    // Remove it
+                    MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, interVpnLinkIid);
+                    // Create it again, but first we have to wait for everything to be removed from dataplane
+                    // TODO: the wait causes an IllegalMonitorStateException
+                    Long timeToWait = Long.getLong("wait.time.sync.install", 1500L);
+                    try {
+                        Thread.sleep(timeToWait);
+                    } catch (InterruptedException e) {
+                        logger.warn("Interrupted while waiting for Flows removal sync.", e);
+                    }
+
+                    InterVpnLink interVpnLink2 = new InterVpnLinkBuilder(interVpnLink).build();
+                    WriteTransaction tx = broker.newWriteOnlyTransaction();
+                    tx.put(LogicalDatastoreType.CONFIGURATION, interVpnLinkIid, interVpnLink2, true);
+                    CheckedFuture<Void, TransactionCommitFailedException> futures = tx.submit();
+                    result.add(futures);
+                }
+            }
+
+            return result;
+        }
+
+    }
+
+}
diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkUtil.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/intervpnlink/InterVpnLinkUtil.java
new file mode 100644 (file)
index 0000000..4b78c19
--- /dev/null
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.netvirt.vpnmanager.intervpnlink;
+
+import com.google.common.base.Optional;
+import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.genius.mdsalutil.*;
+import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
+import org.opendaylight.netvirt.vpnmanager.VpnConstants;
+import org.opendaylight.netvirt.vpnmanager.VpnUtil;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinkStates;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.InterVpnLinks;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkStateBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.FirstEndpointState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.inter.vpn.link.state.SecondEndpointState;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLinkKey;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * This class contains methods to be used as utilities related with inter-vpn-link.
+ *
+ */
+public class InterVpnLinkUtil {
+    private static final Logger LOG = LoggerFactory.getLogger(InterVpnLinkUtil.class);
+
+    /**
+     * Updates VpnToDpn map by adding a fake VpnInterface related to an
+     * InterVpnLink in the corresponding DPNs
+     *
+     * @param broker dataBroker service reference
+     * @param dpnList List of DPNs where the fake InterVpnLink interface must
+     *     be added
+     * @param vpnUuid UUID of the VPN to which the fake interfaces belong
+     */
+    public static void updateVpnToDpnMap(DataBroker broker, List<BigInteger> dpnList, Uuid vpnUuid) {
+        String rd = VpnUtil.getVpnRd(broker, vpnUuid.getValue());
+        InstanceIdentifier<VpnInstanceOpDataEntry> id = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
+        Optional<VpnInstanceOpDataEntry> vpnInstOpData = MDSALUtil.read(broker, LogicalDatastoreType.OPERATIONAL, id);
+        if ( vpnInstOpData.isPresent() ) {
+            for (BigInteger dpnId : dpnList) {
+                String linkIfaceName = String.format("InterVpnLink.%s.%s", vpnUuid.getValue(), dpnId.toString());
+                VpnUtil.mergeDpnInVpnToDpnMap(broker, vpnInstOpData.get(), dpnId,
+                                              Arrays.asList(linkIfaceName));
+            }
+        }
+    }
+
+
+    /**
+     * Retrieves the InterVpnLink object searching by its name
+     *
+     * @param broker dataBroker service reference
+     * @param vpnLinkName Name of the InterVpnLink
+     * @return the InterVpnLink or Optional.absent() if there is no
+     *     InterVpnLink with the specified name
+     */
+    public static Optional<InterVpnLink> getInterVpnLinkByName(DataBroker broker, String vpnLinkName) {
+        InstanceIdentifier<InterVpnLink> interVpnLinksIid =
+                InstanceIdentifier.builder(InterVpnLinks.class)
+                                  .child(InterVpnLink.class, new InterVpnLinkKey(vpnLinkName)).build();
+        return  VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, interVpnLinksIid);
+    }
+
+    /**
+     * Updates inter-VPN link state
+     *
+     * @param broker dataBroker service reference
+     * @param vpnLinkName The name of the InterVpnLink
+     * @param state Sets the state of the InterVpnLink to Active or Error
+     * @param newFirstEndpointState Updates the lportTag and/or DPNs of the 1st
+     *     endpoint of the InterVpnLink
+     * @param newSecondEndpointState Updates the lportTag and/or DPNs of the
+     *     2nd endpoint of the InterVpnLink
+     */
+    public static void updateInterVpnLinkState(DataBroker broker, String vpnLinkName, InterVpnLinkState.State state,
+                                               FirstEndpointState newFirstEndpointState,
+                                               SecondEndpointState newSecondEndpointState) {
+        InterVpnLinkState oldVpnLinkState = VpnUtil.getInterVpnLinkState(broker, vpnLinkName);
+        InterVpnLinkState newVpnLinkState =
+                new InterVpnLinkStateBuilder(oldVpnLinkState).setState(state)
+                                                             .setFirstEndpointState(newFirstEndpointState)
+                                                             .setSecondEndpointState(newSecondEndpointState).build();
+        VpnUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, VpnUtil.getInterVpnLinkStateIid(vpnLinkName),
+                           newVpnLinkState);
+    }
+
+    /**
+     * Installs a Flow in LPortDispatcher table that matches on SI=2 and
+     * the lportTag of one InterVpnLink's endpoint and sets the vrfTag of the
+     * other endpoint and sends to FIB table
+     *
+     * @param broker dataBroker service reference
+     * @param mdsalManager MDSAL API accessor
+     * @param interVpnLink Object that holds the needed information about both
+     *     endpoints of the InterVpnLink.
+     * @param dpnList The list of DPNs where this flow must be installed
+     * @param vpnUuidOtherEndpoint UUID of the other endpoint of the
+     *     InterVpnLink
+     * @param lPortTagOfOtherEndpoint Dataplane identifier of the other
+     *     endpoint of the InterVpnLink
+     */
+    public static void installLPortDispatcherTableFlow(DataBroker broker, IMdsalApiManager mdsalManager,
+                                                       InterVpnLink interVpnLink, List<BigInteger> dpnList,
+                                                       Uuid vpnUuidOtherEndpoint, Integer lPortTagOfOtherEndpoint) {
+        long vpnId = VpnUtil.getVpnId(broker, vpnUuidOtherEndpoint.getValue());
+        for ( BigInteger dpnId : dpnList ) {
+            // insert into LPortDispatcher table
+            Flow lPortDispatcherFlow = buildLPortDispatcherFlow(interVpnLink.getName(), vpnId, lPortTagOfOtherEndpoint);
+            mdsalManager.installFlow(dpnId, lPortDispatcherFlow);
+        }
+    }
+
+   /**
+    * Builds a Flow to be installed into LPortDispatcher table, that matches on
+    * SI=2 + vpnLinkEndpointPseudoPortTag and sends to FIB
+    *
+    * @param interVpnLinkName The name of the InterVpnLink
+    * @param vpnId Dataplane identifier of the VPN, the Vrf Tag.
+    * @param lportTag DataPlane identifier of the LogicalPort.
+    * @return the Flow ready to be installed
+    */
+   public static Flow buildLPortDispatcherFlow(String interVpnLinkName, long vpnId, Integer lportTag) {
+       LOG.info("Inter-vpn-link : buildLPortDispatcherFlow. vpnId {}   lportTag {} ", vpnId, lportTag);
+       List<MatchInfo> matches = Arrays.asList(new MatchInfo(MatchFieldType.metadata,
+                                                              new BigInteger[] {
+                                                                  MetaDataUtil.getMetaDataForLPortDispatcher(lportTag,
+                                                                                VpnConstants.L3VPN_SERVICE_IDENTIFIER),
+                                                                  MetaDataUtil.getMetaDataMaskForLPortDispatcher() }));
+       String flowRef = getLportDispatcherFlowRef(interVpnLinkName, lportTag);
+       Flow lPortDispatcherFlow = MDSALUtil.buildFlowNew(VpnConstants.LPORT_DISPATCHER_TABLE, flowRef,
+                                                         VpnConstants.DEFAULT_LPORT_DISPATCHER_FLOW_PRIORITY, flowRef,
+                                                         0, 0, VpnUtil.getCookieL3((int) vpnId), matches,
+                                                         buildLportDispatcherTableInstructions(vpnId));
+       return lPortDispatcherFlow;
+   }
+
+   /**
+   * Builds a flowRef to be assigned to the flow to be installed into
+   * LPortDispatcher table
+   *
+   * @param interVpnLinkName The name of the InterVpnLink
+   * @param lportTag Dataplane identifier of the LogicalPort
+   * @return the flow reference string
+   */
+   public static String getLportDispatcherFlowRef(String interVpnLinkName, Integer lportTag) {
+       String flowRef = new StringBuffer().append(VpnConstants.FLOWID_PREFIX).append("INTERVPNLINK")
+                     .append(NwConstants.FLOWID_SEPARATOR).append(interVpnLinkName)
+                     .append(NwConstants.FLOWID_SEPARATOR).append(lportTag)
+                     .append(NwConstants.FLOWID_SEPARATOR).append(VpnConstants.L3VPN_SERVICE_IDENTIFIER)
+                     .append(NwConstants.FLOWID_SEPARATOR).append(VpnConstants.DEFAULT_LPORT_DISPATCHER_FLOW_PRIORITY)
+                     .toString();
+       return flowRef;
+   }
+
+
+   public static List<Instruction> buildLportDispatcherTableInstructions (long vpnId) {
+       int instructionKey = 0;
+       List<Instruction> instructions = new ArrayList<Instruction>();
+       instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(BigInteger.valueOf(vpnId),
+                                                                    MetaDataUtil.METADATA_MASK_VRFID,
+                                                                    ++instructionKey));
+       instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(VpnConstants.FIB_TABLE, ++instructionKey));
+
+       return instructions;
+   }
+
+   /**
+    * Retrieves the States of all InterVpnLinks
+    *
+    * @param broker dataBroker service reference
+    * @return the list of objects that holds the InterVpnLink state information
+    */
+   public static List<InterVpnLinkState> getAllInterVpnLinkState(DataBroker broker) {
+       InstanceIdentifier<InterVpnLinkStates> interVpnLinkStateIid = InstanceIdentifier.builder(InterVpnLinkStates.class).build();
+
+       Optional<InterVpnLinkStates> interVpnLinkStateOpData = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION,
+               interVpnLinkStateIid);
+
+       return (interVpnLinkStateOpData.isPresent()) ? interVpnLinkStateOpData.get().getInterVpnLinkState()
+               : new ArrayList<InterVpnLinkState>();
+   }
+
+
+}
index 7b3fc6e9ea8239c7a052f5e95a616fd0d33d2dd0..dc6d140013e5d077e69aafc819ddf50d86cc177d 100644 (file)
@@ -11,6 +11,7 @@ import org.opendaylight.netvirt.vpnmanager.VpnserviceProvider;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 
 public class VpnserviceImplModule extends org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.vpnservice.impl.rev150216.AbstractVpnserviceImplModule {
     public VpnserviceImplModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
@@ -30,14 +31,19 @@ public class VpnserviceImplModule extends org.opendaylight.yang.gen.v1.urn.opend
     public java.lang.AutoCloseable createInstance() {
         IdManagerService idManager = getRpcregistryDependency().getRpcService(IdManagerService.class);
         OdlArputilService arpManager =  getRpcregistryDependency().getRpcService(OdlArputilService.class);
-        OdlInterfaceRpcService interfaceManager = getRpcregistryDependency().getRpcService(OdlInterfaceRpcService.class);
+        OdlInterfaceRpcService odlInterfaceRpcService = getRpcregistryDependency().getRpcService(OdlInterfaceRpcService.class);
+        ItmRpcService itmRpcService = getRpcregistryDependency().getRpcService(ItmRpcService.class);
+
         VpnserviceProvider provider = new VpnserviceProvider();
         provider.setNotificationService(getNotificationServiceDependency());
         provider.setBgpManager(getBgpmanagerDependency());
         provider.setMdsalManager(getMdsalutilDependency());
-        provider.setInterfaceManager(interfaceManager);
+        provider.setOdlInterfaceRpcService(odlInterfaceRpcService);
         provider.setIdManager(idManager);
         provider.setArpManager(arpManager);
+        provider.setITMProvider(itmRpcService);
+        provider.setRpcProviderRegistry(getRpcregistryDependency());
+        provider.setNotificationPublishService(getNotificationPublishServiceDependency());
         getBrokerDependency().registerProvider(provider);
         return provider;
     }
index e31cb9931e101dafded40e4f2970eac1ec8a7465..46f703e0cb04af748661251baec9c325ea1d3cf7 100644 (file)
@@ -67,6 +67,14 @@ module vpnservice-impl {
                     }
                 }
             }
+            container notification-publish-service {
+                uses config:service-ref {
+                    refine type {
+                        mandatory true;
+                        config:required-identity md-sal-binding-impl:binding-new-notification-publish-service;
+                    }
+                }
+            }
         }
     }
 }
index 7e9623fa860ea307660008d3793a32ec43d2cd91..de46e61ae77e15f890cbde3b23667aca5027c4e3 100644 (file)
@@ -374,7 +374,7 @@ public class VpnSubnetRouteHandlerTest {
 
         vpnSubnetRouteHandler.onSubnetUpdatedInVpn(subnetUpdatedInVpn);
 
-        verify(mockWriteTx).delete(LogicalDatastoreType.OPERATIONAL, instPortOp);
+        verify(mockWriteTx).delete(LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
         verify(mockWriteTx).delete(LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 
     }
@@ -384,7 +384,7 @@ public class VpnSubnetRouteHandlerTest {
 
         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetDeletedFromVpn);
 
-        verify(mockWriteTx).delete(LogicalDatastoreType.OPERATIONAL, instPortOp);
+        verify(mockWriteTx).delete(LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
         verify(mockWriteTx).delete(LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 
     }