Bug 6786: L3VPN is not honoring VTEP add or delete in operational cloud
[netvirt.git] / vpnservice / vpnmanager / vpnmanager-impl / src / main / java / org / opendaylight / netvirt / vpnmanager / VpnSubnetRouteHandler.java
index 796057cd56cb130e8464eb8c3035090e09d9b9a9..438b33ec85ecbf53b484fa7c1b5cedcf0b36fef5 100644 (file)
@@ -7,61 +7,71 @@
  */
 package org.opendaylight.netvirt.vpnmanager;
 
-import java.util.*;
-
+import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
-
-import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.genius.mdsalutil.MDSALUtil;
+import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.*;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.subnet.to.dpn.VpnInterfaces;
-import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnListener;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.PortAddedToSubnet;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.PortRemovedFromSubnet;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterAssociatedToVpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterDisassociatedFromVpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetAddedToVpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetAddedToVpnBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetDeletedFromVpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetDeletedFromVpnBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetUpdatedInVpn;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
-
-import java.math.BigInteger;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Optional;
-
-
 public class VpnSubnetRouteHandler implements NeutronvpnListener {
     private static final Logger logger = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
-
-    private final DataBroker broker;
-    private SubnetOpDpnManager subOpDpnManager;
+    private final DataBroker dataBroker;
+    private final SubnetOpDpnManager subOpDpnManager;
     private final IBgpManager bgpManager;
-    private IdManagerService idManager;
-    private VpnInterfaceManager vpnInterfaceManager;
-
-    public VpnSubnetRouteHandler(final DataBroker db, IBgpManager bgpManager, VpnInterfaceManager vpnIntfManager) {
-        broker = db;
-        subOpDpnManager = new SubnetOpDpnManager(broker);
+    private final VpnInterfaceManager vpnInterfaceManager;
+    private final IdManagerService idManager;
+    private LockManagerService lockManager;
+
+    public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
+                                 final IBgpManager bgpManager, final VpnInterfaceManager vpnIntfManager,
+                                 final IdManagerService idManager, LockManagerService lockManagerService) {
+        this.dataBroker = dataBroker;
+        this.subOpDpnManager = subnetOpDpnManager;
         this.bgpManager = bgpManager;
         this.vpnInterfaceManager = vpnIntfManager;
-    }
-
-    public void setIdManager(IdManagerService idManager) {
         this.idManager = idManager;
+        this.lockManager = lockManagerService;
     }
 
     @Override
@@ -81,15 +91,26 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 
         logger.info("onSubnetAddedToVpn: Subnet " + subnetId.getValue() + " being added to vpn");
+        long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
+        if (vpnId == VpnConstants.INVALID_ID) {
+            vpnInterfaceManager.waitForVpnInstance(vpnName, VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS, vpnInterfaceManager.getvpnInstanceToIdSynchronizerMap());
+            vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
+            if (vpnId == VpnConstants.INVALID_ID) {
+                logger.error("onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} processing subnet {} with IP {} " +
+                        ", bailing out now.", vpnName, subnetId, subnetIp);
+                return;
+            }
+        }
         //TODO(vivek): Change this to use more granularized lock at subnetId level
-        synchronized (this) {
+        try {
+            VpnUtil.lockSubnet(lockManager, subnetId.getValue());
             try {
                 Subnetmap subMap = null;
 
                 // Please check if subnetId belongs to an External Network
                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
-                Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
+                Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
                 if (!sm.isPresent()) {
                     logger.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
                     return;
@@ -97,7 +118,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 subMap = sm.get();
                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class).
                         child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
-                Optional<Networks> optionalNets = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, netsIdentifier);
+                Optional<Networks> optionalNets = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, netsIdentifier);
                 if (optionalNets.isPresent()) {
                     logger.info("onSubnetAddedToVpn: subnet {} is an external subnet on external network {}, so ignoring this for SubnetRoute",
                             subnetId.getValue(), subMap.getNetworkId().getValue());
@@ -106,7 +127,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
-                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
+                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
                         LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (optionalSubs.isPresent()) {
@@ -124,7 +145,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
                 subOpBuilder.setSubnetId(subnetId);
                 subOpBuilder.setSubnetCidr(subnetIp);
-                String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(broker, vpnName);
+                String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(dataBroker, vpnName);
                 if (rd == null) {
                     logger.error("onSubnetAddedToVpn: The VPN Instance name " + notification.getVpnName() + " does not have RD ");
                     return;
@@ -139,10 +160,17 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 List<Uuid> portList = subMap.getPortList();
                 if (portList != null) {
                     for (Uuid port: portList) {
-                        Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,port.getValue());
+                        Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
                         if (intfState != null) {
-                            dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
-                            if (dpnId == null) {
+                            try {
+                                dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
+                            } catch (Exception e) {
+                                logger.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
+                                        " subnetroute inclusion for this interface failed with exception {}",
+                                        port.getValue(), e);
+                                continue;
+                            }
+                            if (dpnId.equals(BigInteger.ZERO)) {
                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not assigned DPN yet, ignoring ");
                                 continue;
                             }
@@ -177,8 +205,11 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                         And also advertise the subnet route entry via BGP.
                         */
                         int label = getLabel(rd, subnetIp);
-                        addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
-                        advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
+                        if (label == 0) {
+                            logger.error("Unable to fetch label from Id Manager. Bailing out of handling addition of subnet {} to vpn {}", subnetIp, vpnName);
+                            return;
+                        }
+                        addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
                         subOpBuilder.setRouteAdvState(TaskState.Done);
                     } catch (Exception ex) {
                         logger.error("onSubnetAddedToVpn: FIB rules and Advertising nhDpnId " + nhDpnId +
@@ -190,14 +221,19 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 }
 
                 subOpEntry = subOpBuilder.build();
-                MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
-                logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet " + subnetId.getValue());
+                MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
+                logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}",
+                        subnetId.getValue());
             } catch (Exception ex) {
                 logger.error("Creation of SubnetOpDataEntry for subnet " +
                         subnetId.getValue() + " failed {}", ex);
             } finally {
+                VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
             }
+        } catch (Exception e) {
+            logger.error("Unable to handle subnet {} added to vpn {} {}", subnetIp, vpnName, e);
         }
+
     }
 
     @Override
@@ -209,12 +245,13 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         }
         logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed from vpn");
         //TODO(vivek): Change this to use more granularized lock at subnetId level
-        synchronized (this) {
+        try {
+            VpnUtil.lockSubnet(lockManager, subnetId.getValue());
             try {
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
-                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
+                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
                         LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (!optionalSubs.isPresent()) {
@@ -230,7 +267,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                  */
                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
-                Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
+                Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
                 if (!sm.isPresent()) {
                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
                 } else {
@@ -241,7 +278,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                             InstanceIdentifier<PortOpDataEntry> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).
                                     child(PortOpDataEntry.class, new PortOpDataEntryKey(port.getValue())).build();
                             logger.trace("Deleting portOpData entry for port " + port.getValue());
-                            MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
+                            MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
                         }
                     }
                 }
@@ -250,13 +287,13 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 String rd = subOpBuilder.getVrfId();
                 String subnetIp = subOpBuilder.getSubnetCidr();
                 String vpnName = subOpBuilder.getVpnName();
-                MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
+                BigInteger nhDpnId = subOpBuilder.getNhDpnId();
+                MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
                 logger.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore", subnetId.getValue());
                 try {
                     //Withdraw the routes for all the interfaces on this subnet
                     //Remove subnet route entry from FIB
                     deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
-                    withdrawSubnetRoutefromBgp(rd, subnetIp);
                 } catch (Exception ex) {
                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
                             subnetId.getValue() + " failed {}" + ex);
@@ -265,7 +302,10 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 logger.error("Removal of SubnetOpDataEntry for subnet " +
                         subnetId.getValue() + " failed {}" + ex);
             } finally {
+                VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
             }
+        } catch (Exception e) {
+            logger.error("Unable to handle subnet {} removed to vpn {} {}", notification.getSubnetIp(), notification.getVpnName(), e);
         }
     }
 
@@ -283,7 +323,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
 
         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                 child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
-        Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
+        Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
                 LogicalDatastoreType.OPERATIONAL,
                 subOpIdentifier);
         if (optionalSubs.isPresent()) {
@@ -310,26 +350,35 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
 
         logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
         //TODO(vivek): Change this to use more granularized lock at subnetId level
-        synchronized (this) {
+        try {
+            VpnUtil.lockSubnet(lockManager, subnetId.getValue());
             try {
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 
-                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (!optionalSubs.isPresent()) {
                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
                             " that is not in VPN, ignoring");
                     return;
                 }
-                Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,portId.getValue());
+                Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
                 if (intfState == null) {
                     // Interface State not yet available
                     subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
                     return;
                 }
-                BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
-                if (dpnId == null) {
+                BigInteger dpnId = BigInteger.ZERO;
+                try {
+                    dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
+                } catch (Exception e) {
+                    logger.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
+                            " subnetroute inclusion for this interface failed with exception {}",
+                            portId.getValue(), e);
+                    return;
+                }
+                if (dpnId.equals(BigInteger.ZERO)) {
                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
                     return;
                 }
@@ -361,8 +410,11 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                         // Write the Subnet Route Entry to FIB
                         // Advertise BGP Route here and set route_adv_state to DONE
                         int label = getLabel(rd, subnetIp);
-                        addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
-                        advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
+                        if (label == 0) {
+                            logger.error("Unable to fetch label from Id Manager. Bailing out of handling addition of port {} to subnet {} in vpn {}", portId.getValue(), subnetIp, vpnName);
+                            return;
+                        }
+                        addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
                         subOpBuilder.setRouteAdvState(TaskState.Done);
                     } catch (Exception ex) {
                         logger.error("onPortAddedToSubnet: Advertising NextHopDPN "+ nhDpnId +
@@ -370,14 +422,17 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                     }
                 }
                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
-                MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
+                MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
                 logger.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port " + portId.getValue());
 
             } catch (Exception ex) {
                 logger.error("Creation of SubnetOpDataEntry for subnet " +
                         subnetId.getValue() + " failed {}", ex);
             } finally {
+                VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
             }
+        } catch (Exception e) {
+            logger.error("Unable to handle port {} added to subnet {} {}", portId.getValue(), subnetId.getValue(), e);
         }
     }
 
@@ -388,7 +443,8 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
 
         logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
         //TODO(vivek): Change this to use more granularized lock at subnetId level
-        synchronized (this) {
+        try {
+            VpnUtil.lockSubnet(lockManager, subnetId.getValue());
             try {
                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
                 if (portOpEntry == null) {
@@ -403,7 +459,7 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
-                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (!optionalSubs.isPresent()) {
                     logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
@@ -429,7 +485,6 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                             try {
                                 // withdraw route from BGP
                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
-                                withdrawSubnetRoutefromBgp(rd, subnetIp);
                                 subOpBuilder.setRouteAdvState(TaskState.Na);
                             } catch (Exception ex) {
                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
@@ -445,8 +500,11 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                                 // Advertise the new NexthopIP to BGP for this subnet
                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
                                 int label = getLabel(rd, subnetIp);
-                                addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
-                                advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
+                                if (label == 0) {
+                                    logger.error("Unable to fetch label from Id Manager. Bailing out of handling removal  of port {} from subnet {} in vpn {}", portId.getValue(), subnetIp, vpnName);
+                                    return;
+                                }
+                                addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
                                 subOpBuilder.setRouteAdvState(TaskState.Done);
                             } catch (Exception ex) {
                                 logger.error("onPortRemovedFromSubnet: Swapping Withdrawing NextHopDPN " + dpnId +
@@ -458,39 +516,42 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                     }
                 }
                 subOpEntry = subOpBuilder.build();
-                MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
+                MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
                 logger.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port " + portId.getValue());
             } catch (Exception ex) {
                 logger.error("Creation of SubnetOpDataEntry for subnet " +
                         subnetId.getValue() + " failed {}" + ex);
             } finally {
+                VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
             }
+        } catch (Exception e) {
+            logger.error("Unable to handle port {} removed from subnet {} {}", portId.getValue(), subnetId.getValue(), e);
         }
     }
 
     public void onInterfaceUp(BigInteger dpnId, String intfName) {
         logger.info("onInterfaceUp: Port " + intfName);
         //TODO(vivek): Change this to use more granularized lock at subnetId level
-        synchronized (this) {
-            SubnetToDpn subDpn = null;
-            PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
-            if (portOpEntry == null) {
-                logger.info("onInterfaceUp: Port " + intfName  + "is part of a subnet not in VPN, ignoring");
-                return;
-            }
-
-            if ((dpnId == null) || (dpnId == BigInteger.ZERO)) {
-                dpnId = portOpEntry.getDpnId();
-                if (dpnId == null) {
-                    logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
-                    return;
-                }
+        SubnetToDpn subDpn = null;
+        PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
+        if (portOpEntry == null) {
+            logger.info("onInterfaceUp: Port " + intfName  + "is part of a subnet not in VPN, ignoring");
+            return;
+        }
+        if ((dpnId == null) || (dpnId == BigInteger.ZERO)) {
+            dpnId = portOpEntry.getDpnId();
+            if (dpnId == null) {
+                logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
+            return;
             }
-            Uuid subnetId = portOpEntry.getSubnetId();
+        }
+        Uuid subnetId = portOpEntry.getSubnetId();
+        try {
+            VpnUtil.lockSubnet(lockManager, subnetId.getValue());
             try {
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
-                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
+                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (!optionalSubs.isPresent()) {
                     logger.error("onInterfaceUp: SubnetOpDataEntry for subnet " + subnetId.getValue() +
@@ -521,45 +582,52 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                         // Write the Subnet Route Entry to FIB
                         // Advertise BGP Route here and set route_adv_state to DONE
                         int label = getLabel(rd, subnetIp);
-                        addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
-                        advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
+                        if (label == 0) {
+                            logger.error("Unable to fetch label from Id Manager. Bailing out of handling interface up event for port {} for subnet {} in vpn {}", intfName, subnetIp, vpnName);
+                            return;
+                        }
+                        addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
                         subOpBuilder.setRouteAdvState(TaskState.Done);
                     } catch (Exception ex) {
                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
-                          subnetId.getValue() + " to BGP failed {}" + ex);
+                                subnetId.getValue() + " to BGP failed {}" + ex);
                     }
                 }
                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
-                MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
+                MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
                 logger.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
             } catch (Exception ex) {
                 logger.error("Creation of SubnetOpDataEntry for subnet " +
                         subnetId.getValue() + " failed {}" + ex);
             } finally {
+                VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
             }
+        } catch (Exception e) {
+            logger.error("Unable to handle interface up event for port {} in subnet {} {}", portOpEntry.getPortId(), subnetId.getValue(), e);
         }
     }
 
     public void onInterfaceDown(final BigInteger dpnId, final String interfaceName) {
         logger.info("onInterfaceDown: Port " + interfaceName);
         //TODO(vivek): Change this to use more granularized lock at subnetId level
-        synchronized (this) {
-            PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(interfaceName);
-            if (portOpEntry == null) {
-                logger.info("onInterfaceDown: Port " + interfaceName  + "is part of a subnet not in VPN, ignoring");
-                return;
-            }
-            if ((dpnId  == null) ||(dpnId == BigInteger.ZERO)) {
-                logger.error("onInterfaceDown: Unable to determine the DPNID for port " + interfaceName);
-                return;
-            }
-            Uuid subnetId = portOpEntry.getSubnetId();
+        PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(interfaceName);
+        if (portOpEntry == null) {
+            logger.info("onInterfaceDown: Port " + interfaceName  + "is part of a subnet not in VPN, ignoring");
+            return;
+        }
+        if ((dpnId  == null) ||(dpnId == BigInteger.ZERO)) {
+            logger.error("onInterfaceDown: Unable to determine the DPNID for port " + interfaceName);
+            return;
+        }
+        Uuid subnetId = portOpEntry.getSubnetId();
+        try {
+            VpnUtil.lockSubnet(lockManager, subnetId.getValue());
             try {
                 logger.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
-                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
+                Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
                         LogicalDatastoreType.OPERATIONAL,
                         subOpIdentifier);
                 if (!optionalSubs.isPresent()) {
@@ -586,7 +654,6 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                             try {
                                 // Withdraw route from BGP for this subnet
                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
-                                withdrawSubnetRoutefromBgp(rd, subnetIp);
                                 subOpBuilder.setRouteAdvState(TaskState.Na);
                             } catch (Exception ex) {
                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
@@ -601,8 +668,11 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                                 // Best effort Withdrawal of route from BGP for this subnet
                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
                                 int label = getLabel(rd, subnetIp);
-                                addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
-                                advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
+                                if (label == 0) {
+                                    logger.error("Unable to fetch label from Id Manager. Bailing out of handling interface down event for port {} in subnet {} for vpn {}", interfaceName, subnetIp, vpnName);
+                                    return;
+                                }
+                                addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
                                 subOpBuilder.setRouteAdvState(TaskState.Done);
                             } catch (Exception ex) {
                                 logger.error("onInterfaceDown: Swapping Withdrawing NextHopDPN " + dpnId + " information for subnet " +
@@ -613,13 +683,16 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
                     }
                 }
                 subOpEntry = subOpBuilder.build();
-                MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
+                MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
                 logger.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore port down " + interfaceName);
             } catch (Exception ex) {
                 logger.error("Creation of SubnetOpDataEntry for subnet " +
                         subnetId.getValue() + " failed {}" + ex);
             } finally {
+                VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
             }
+        } catch (Exception e) {
+            logger.error("Unable to handle interface down event for port {} in subnet {} {}", portOpEntry.getPortId(), subnetId.getValue(), e);
         }
     }
 
@@ -632,16 +705,33 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
     }
 
     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
-                                     Long elanTag, int label) {
+                                     Long elanTag, int label, Uuid subnetId) throws Exception {
         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
-        String nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
-        if(nexthopIp != null)
-            vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, vpnName, subnetIp, nexthopIp, label, elanTag, nhDpnId , null);
-        else
-            logger.info("Unable to get nextHop ip address for nextHop DPN {}. Abort adding subnet route to FIB table.", nhDpnId);
+        String nexthopIp = null;
+        try {
+            nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
+        } catch (Exception e) {
+            logger.warn("Unable to find nexthopip for subnetroute subnetip {}", subnetIp);
+            return;
+        }
+        if (nexthopIp != null) {
+            VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker, vpnName), subnetIp), VpnUtil.getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp));
+            vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, vpnName, subnetIp, nexthopIp, label, elanTag, nhDpnId, null);
+            try {
+                // BGPManager (inside ODL) requires a withdraw followed by advertise
+                // due to bugs with ClusterDataChangeListener used by BGPManager.
+                //bgpManager.withdrawPrefix(rd, subnetIp);
+                bgpManager.advertisePrefix(rd, subnetIp, Arrays.asList(nexthopIp), label);
+            } catch (Exception e) {
+                logger.error("Fail: Subnet route not advertised for rd {} subnetIp {}", rd, subnetIp, e);
+                throw e;
+            }
+        } else {
+            logger.warn("The nexthopip is empty for subnetroute subnetip {}, ignoring fib route addition", subnetIp);
+        }
     }
 
     private int getLabel(String rd, String subnetIp) {
@@ -651,43 +741,14 @@ public class VpnSubnetRouteHandler implements NeutronvpnListener {
         return label;
     }
 
-    private void deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName) {
+    private void deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName) throws Exception {
         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
         vpnInterfaceManager.deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
-    }
-
-    private void advertiseSubnetRouteToBgp(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
-                                           Long elanTag, int label) throws Exception {
-        Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
-        Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
-        Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
-        Preconditions.checkNotNull(nhDpnId, "nhDpnId cannot be null or empty!");
-        Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
-        String nexthopIp = null;
-        nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
-        if (nexthopIp == null) {
-            logger.error("createSubnetRouteInVpn: Unable to obtain endpointIp address for DPNId " + nhDpnId);
-            throw new Exception("Unable to obtain endpointIp address for DPNId " + nhDpnId);
-        }
-        try {
-            // BGPManager (inside ODL) requires a withdraw followed by advertise
-            // due to bugs with ClusterDataChangeListener used by BGPManager.
-            //bgpManager.withdrawPrefix(rd, subnetIp);
-            bgpManager.advertisePrefix(rd, subnetIp, Arrays.asList(nexthopIp), label);
-        } catch (Exception e) {
-            logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
-            throw e;
-        }
-    }
-
-    private void withdrawSubnetRoutefromBgp(String rd, String subnetIp) throws Exception {
-        Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
-        Preconditions.checkNotNull(subnetIp, "SubnetIp cannot be null or empty!");
         try {
             bgpManager.withdrawPrefix(rd, subnetIp);
         } catch (Exception e) {
-            logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
+            logger.error("Fail: Subnet route not withdrawn for rd {} subnetIp {}", rd, subnetIp, e);
             throw e;
         }
     }