Dualstack support for L3VPN - single router Dual stack 46/60246/75
authorPhilippe Guibert <philippe.guibert@6wind.com>
Tue, 11 Jul 2017 12:40:15 +0000 (13:40 +0100)
committerPhilippe Guibert <philippe.guibert@6wind.com>
Mon, 18 Sep 2017 09:02:03 +0000 (10:02 +0100)
According to dualstack specification [0], this commit introduces changes
related to "one router solution" case.

* NeutronPortChangeListener: Create only one VPN interface instance
  for one neutron port. This instance can include a list of IPv4 and/or
  IPv6 subnetMaps, where addresses where allocated for spawned port.

* NeutronvpnManager: in createVpnInterface we no longer use routerId as
  argument, instead we will iterate over all port IP addresses to create
  according adjacencies objects and than, we pass composed adjacencies
  list to writeVpnInterfaceToDs method.

Change-Id: I5f527be0ebea8f1a74e9a790aa9c3fec6291bda7
Signed-off-by: Valentina Krasnobaeva <valentina.krasnobaeva@6wind.com>
Signed-off-by: Philippe Guibert <philippe.guibert@6wind.com>
Signed-off-by: Noel de Prandieres <prandieres@6wind.com>
vpnservice/neutronvpn/neutronvpn-impl/src/main/java/org/opendaylight/netvirt/neutronvpn/NeutronPortChangeListener.java
vpnservice/neutronvpn/neutronvpn-impl/src/main/java/org/opendaylight/netvirt/neutronvpn/NeutronvpnManager.java

index df9d60f246212a7578b0261c8c39e9ee883d7d89..639193b06a26e745e8379e55c588fdc81e0d4c92 100644 (file)
@@ -18,7 +18,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-
+import java.util.stream.Collectors;
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.inject.Singleton;
@@ -114,7 +114,6 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
             return;
         }
         NeutronvpnUtils.addToPortCache(input);
-
         String portStatus = NeutronUtils.PORT_STATUS_DOWN;
         if (!Strings.isNullOrEmpty(input.getDeviceOwner()) && !Strings.isNullOrEmpty(input.getDeviceId())) {
             if (input.getDeviceOwner().equals(NeutronConstants.DEVICE_OWNER_ROUTER_INF)) {
@@ -133,7 +132,6 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
         if (input.getFixedIps() != null && !input.getFixedIps().isEmpty()) {
             handleNeutronPortCreated(input);
         }
-
         NeutronUtils.createPortStatus(input.getUuid().getValue(), portStatus, dataBroker);
     }
 
@@ -172,10 +170,10 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
     @SuppressWarnings("checkstyle:IllegalCatch")
     protected void update(InstanceIdentifier<Port> identifier, Port original, Port update) {
         final String portName = update.getUuid().getValue();
-        LOG.trace("Updating Port : key: {}, original value={}, update value={}", identifier, original, update);
         Network network = NeutronvpnUtils.getNeutronNetwork(dataBroker, update.getNetworkId());
+        LOG.info("Update port {} from network {}", portName, update.getNetworkId().toString());
         if (network == null || !NeutronvpnUtils.isNetworkTypeSupported(network)) {
-            LOG.warn("neutron vpn received a port update() for a network without a provider extension augmentation "
+            LOG.error("neutron vpn received a port update() for a network without a provider extension augmentation "
                     + "or with an unsupported network type for the port {} which is part of network {}",
                     portName, network);
             return;
@@ -259,29 +257,37 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
             elanService.handleKnownL3DmacAddress(routerPort.getMacAddress().getValue(), infNetworkId.getValue(),
                     NwConstants.ADD_FLOW);
             if (existingVpnId == null) {
+                Uuid vpnId = NeutronvpnUtils.getVpnForRouter(dataBroker, routerId, true);
+                List<Subnetmap> subnetMapList = new ArrayList<>();
+                if (vpnId == null) {
+                    vpnId = routerId;
+                }
                 for (FixedIps portIP : routerPort.getFixedIps()) {
-                    Uuid vpnId = NeutronvpnUtils.getVpnForRouter(dataBroker, routerId, true);
-                    if (vpnId == null) {
-                        vpnId = routerId;
-                    }
                     // NOTE:  Please donot change the order of calls to updateSubnetNodeWithFixedIP
                     // and addSubnetToVpn here
                     String ipValue = String.valueOf(portIP.getIpAddress().getValue());
-                    nvpnManager.updateSubnetNodeWithFixedIp(portIP.getSubnetId(), routerId,
+                    Uuid subnetId = portIP.getSubnetId();
+                    nvpnManager.updateSubnetNodeWithFixedIp(subnetId, routerId,
                             routerPort.getUuid(), ipValue, routerPort.getMacAddress().getValue());
-                    nvpnManager.createVpnInterface(vpnId, routerId, routerPort, null);
+                    Subnetmap sn = NeutronvpnUtils.getSubnetmap(dataBroker, subnetId);
+                    subnetMapList.add(sn);
+                }
+                if (! subnetMapList.isEmpty()) {
+                    nvpnManager.createVpnInterface(vpnId, routerPort, null);
+                }
+                for (FixedIps portIP : routerPort.getFixedIps()) {
+                    String ipValue = String.valueOf(portIP.getIpAddress().getValue());
                     nvpnManager.addSubnetToVpn(vpnId, portIP.getSubnetId());
-                    nvpnNatManager.handleSubnetsForExternalRouter(routerId, dataBroker);
-                    WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
-                    String portInterfaceName = createOfPortInterface(routerPort, wrtConfigTxn);
-                    createElanInterface(routerPort, portInterfaceName, wrtConfigTxn);
-                    wrtConfigTxn.submit();
-                    PhysAddress mac = new PhysAddress(routerPort.getMacAddress().getValue());
                     LOG.trace("NeutronPortChangeListener Add Subnet Gateway IP {} MAC {} Interface {} VPN {}",
                             ipValue, routerPort.getMacAddress(),
                             routerPort.getUuid().getValue(), vpnId.getValue());
-
                 }
+                nvpnNatManager.handleSubnetsForExternalRouter(routerId, dataBroker);
+                WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
+                String portInterfaceName = createOfPortInterface(routerPort, wrtConfigTxn);
+                createElanInterface(routerPort, portInterfaceName, wrtConfigTxn);
+                wrtConfigTxn.submit();
+                PhysAddress mac = new PhysAddress(routerPort.getMacAddress().getValue());
             } else {
                 LOG.error("Neutron network {} corresponding to router interface port {} for neutron router {} already"
                     + " associated to VPN {}", infNetworkId.getValue(), routerPort.getUuid().getValue(),
@@ -297,19 +303,24 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
 
             elanService.handleKnownL3DmacAddress(routerPort.getMacAddress().getValue(), infNetworkId.getValue(),
                     NwConstants.DEL_FLOW);
+            List<Subnetmap> subnetMapList = new ArrayList<>();
+            Uuid vpnId = NeutronvpnUtils.getVpnForRouter(dataBroker, routerId, true);
+            if (vpnId == null) {
+                vpnId = routerId;
+            }
+            for (FixedIps portIP : routerPort.getFixedIps()) {
+                Subnetmap sn = NeutronvpnUtils.getSubnetmap(dataBroker, portIP.getSubnetId());
+                subnetMapList.add(sn);
+            }
+            /* Remove ping responder for router interfaces
+             *  A router interface reference in a VPN will have to be removed before the host interface references
+             * for that subnet in the VPN are removed. This is to ensure that the FIB Entry of the router interface
+             *  is not the last entry to be removed for that subnet in the VPN.
+             *  If router interface FIB entry is the last to be removed for a subnet in a VPN , then all the host
+             *  interface references in the vpn will already have been cleared, which will cause failures in
+             *  cleanup of router interface flows*/
+            nvpnManager.deleteVpnInterface(vpnId, routerPort, null);
             for (FixedIps portIP : routerPort.getFixedIps()) {
-                Uuid vpnId = NeutronvpnUtils.getVpnForRouter(dataBroker, routerId, true);
-                if (vpnId == null) {
-                    vpnId = routerId;
-                }
-                /* Remove ping responder for router interfaces
-                *  A router interface reference in a VPN will have to be removed before the host interface references
-                * for that subnet in the VPN are removed. This is to ensure that the FIB Entry of the router interface
-                *  is not the last entry to be removed for that subnet in the VPN.
-                *  If router interface FIB entry is the last to be removed for a subnet in a VPN , then all the host
-                *  interface references in the vpn will already have been cleared, which will cause failures in
-                *  cleanup of router interface flows*/
-                nvpnManager.deleteVpnInterface(vpnId, routerId, routerPort, null);
                 // NOTE:  Please donot change the order of calls to removeSubnetFromVpn and
                 // and updateSubnetNodeWithFixedIP
                 nvpnManager.removeSubnetFromVpn(vpnId, portIP.getSubnetId());
@@ -345,7 +356,8 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
     private void handleNeutronPortCreated(final Port port) {
         final String portName = port.getUuid().getValue();
         final Uuid portId = port.getUuid();
-        final Uuid subnetId = port.getFixedIps().get(0).getSubnetId();
+        // do not check that neutron network includes only 2 different ethertype subnets (IPv4 and IPv6)
+        final List<FixedIps> portIpAddrsList = port.getFixedIps();
         final DataStoreJobCoordinator portDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
         if (NeutronConstants.IS_ODL_DHCP_PORT.test(port)) {
             return;
@@ -353,12 +365,13 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
         portDataStoreCoordinator.enqueueJob("PORT- " + portName, () -> {
             WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
             List<ListenableFuture<Void>> futures = new ArrayList<>();
-
             // add direct port to subnetMaps config DS
             if (!NeutronUtils.isPortVnicTypeNormal(port)) {
-                nvpnManager.updateSubnetmapNodeWithPorts(subnetId, null, portId);
+                for (FixedIps ip: portIpAddrsList) {
+                    nvpnManager.updateSubnetmapNodeWithPorts(ip.getSubnetId(), null, portId);
+                    futures.add(wrtConfigTxn.submit());
+                }
                 LOG.info("Port {} is not a NORMAL VNIC Type port; OF Port interfaces are not created", portName);
-                futures.add(wrtConfigTxn.submit());
                 return futures;
             }
             LOG.info("Of-port-interface creation for port {}", portName);
@@ -366,14 +379,20 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
             String portInterfaceName = createOfPortInterface(port, wrtConfigTxn);
             LOG.debug("Creating ELAN Interface for port {}", portName);
             createElanInterface(port, portInterfaceName, wrtConfigTxn);
-
-            Subnetmap subnetMap = nvpnManager.updateSubnetmapNodeWithPorts(subnetId, portId, null);
-            Uuid vpnId = (subnetMap != null) ? subnetMap.getVpnId() : null;
-            Uuid routerId = (subnetMap != null) ? subnetMap.getRouterId() : null;
+            List<Subnetmap> subnetMapList = new ArrayList<>();
+            Uuid vpnId = null; // vpnId is the same for all neutron-port subnets
+            for (FixedIps ip: portIpAddrsList) {
+                Subnetmap subnetMap = nvpnManager.updateSubnetmapNodeWithPorts(ip.getSubnetId(), portId, null);
+                if (subnetMap != null && subnetMap.getVpnId() != null) {
+                    vpnId = subnetMap.getVpnId();
+                    subnetMapList.add(subnetMap);
+                }
+            }
             if (vpnId != null) {
-                // create vpn-interface on this neutron port
-                LOG.debug("Adding VPN Interface for port {}", portName);
-                nvpnManager.createVpnInterface(vpnId, routerId, port, wrtConfigTxn);
+                // create new vpn-interface for neutron port
+                LOG.debug("handleNeutronPortCreated: Adding VPN Interface for port {} from network {}", portName,
+                           port.getNetworkId().toString());
+                nvpnManager.createVpnInterface(vpnId, port, wrtConfigTxn);
             }
             futures.add(wrtConfigTxn.submit());
             return futures;
@@ -383,29 +402,33 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
     private void handleNeutronPortDeleted(final Port port) {
         final String portName = port.getUuid().getValue();
         final Uuid portId = port.getUuid();
-        final Uuid subnetId = port.getFixedIps().get(0).getSubnetId();
-        if (NeutronConstants.IS_ODL_DHCP_PORT.test(port)) {
-            return;
-        }
+        final List<FixedIps> portIpsList = port.getFixedIps();
         final DataStoreJobCoordinator portDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
         portDataStoreCoordinator.enqueueJob("PORT- " + portName, () -> {
             WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
             List<ListenableFuture<Void>> futures = new ArrayList<>();
-
-            // remove direct port from subnetMaps config DS
             if (!NeutronUtils.isPortVnicTypeNormal(port)) {
-                nvpnManager.removePortsFromSubnetmapNode(subnetId, null, portId);
+                for (FixedIps ip: portIpsList) {
+                    // remove direct port from subnetMaps config DS
+                    nvpnManager.removePortsFromSubnetmapNode(ip.getSubnetId(), null, portId);
+                    futures.add(wrtConfigTxn.submit());
+                }
                 LOG.info("Port {} is not a NORMAL VNIC Type port; OF Port interfaces are not created", portName);
-                futures.add(wrtConfigTxn.submit());
                 return futures;
             }
-            Subnetmap subnetMap = nvpnManager.removePortsFromSubnetmapNode(subnetId, portId, null);
-            Uuid vpnId = (subnetMap != null) ? subnetMap.getVpnId() : null;
-            Uuid routerId = (subnetMap != null) ? subnetMap.getRouterId() : null;
+            Uuid vpnId = null; // vpnId is the same for all neutron-port subnets
+            List<Subnetmap> subnetMapList = new ArrayList<>();
+            for (FixedIps ip: portIpsList) {
+                Subnetmap subnetMap = nvpnManager.removePortsFromSubnetmapNode(ip.getSubnetId(), portId, null);
+                if (subnetMap != null && subnetMap.getVpnId() != null) {
+                    vpnId = subnetMap.getVpnId();
+                    subnetMapList.add(subnetMap);
+                }
+            }
             if (vpnId != null) {
                 // remove vpn-interface for this neutron port
                 LOG.debug("removing VPN Interface for port {}", portName);
-                nvpnManager.deleteVpnInterface(vpnId, routerId, port, wrtConfigTxn);
+                nvpnManager.deleteVpnInterface(vpnId, port, wrtConfigTxn);
             }
             // Remove of-port interface for this neutron port
             // ELAN interface is also implicitly deleted as part of this operation
@@ -418,48 +441,52 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
         });
     }
 
+
     private void handleNeutronPortUpdated(final Port portoriginal, final Port portupdate) {
-        if (portoriginal.getFixedIps() == null || portoriginal.getFixedIps().isEmpty()) {
+        final List<FixedIps> portoriginalIps = portoriginal.getFixedIps();
+        final List<FixedIps> portupdateIps = portupdate.getFixedIps();
+        LOG.trace("PORT ORIGINAL: {} from network {} with fixed IPs: {}; "
+                    + "PORT UPDATE: {} from network {} with fixed IPs: {}",
+                    portoriginal.getName(), portoriginal.getNetworkId().toString(), portoriginalIps.toString(),
+                    portupdate.getName(), portupdate.getNetworkId().toString(), portupdateIps.toString());
+        if (portoriginalIps == null || portoriginalIps.isEmpty()) {
             handleNeutronPortCreated(portupdate);
             return;
         }
-
-        if (portupdate.getFixedIps() == null || portupdate.getFixedIps().isEmpty()) {
-            LOG.debug("Ignoring portUpdate (fixed_ip removal) for port {} as this case is handled "
+        if (portupdateIps == null || portupdateIps.isEmpty()) {
+            LOG.error("Ignoring portUpdate (fixed_ip removal) for port {} as this case is handled "
                       + "during subnet deletion event.", portupdate.getUuid().getValue());
             return;
         }
         final DataStoreJobCoordinator portDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
         portDataStoreCoordinator.enqueueJob("PORT- " + portupdate.getUuid().getValue(), () -> {
             WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
-            Uuid vpnIdOld = null;
-            Uuid vpnIdNew = null;
-            final Uuid oldSubnetId = portoriginal.getFixedIps().get(0).getSubnetId();
-            final Uuid newSubnetId = portupdate.getFixedIps().get(0).getSubnetId();
-            // check if subnet UUID has changed upon change in fixedIP
-            final Boolean subnetUpdated = oldSubnetId.equals(newSubnetId) ? false : true;
-            if (subnetUpdated) {
-                Subnetmap subnetMapOld = nvpnManager.removePortsFromSubnetmapNode(oldSubnetId, portoriginal
-                        .getUuid(), null);
-                vpnIdOld = (subnetMapOld != null) ? subnetMapOld.getVpnId() : null;
-                Subnetmap subnetMapNew = nvpnManager.updateSubnetmapNodeWithPorts(newSubnetId, portupdate
-                                .getUuid(), null);
-                vpnIdNew = (subnetMapNew != null) ? subnetMapNew.getVpnId() : null;
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            final List<Uuid> originalSnMapsIds = portoriginalIps.stream().map(ip -> ip.getSubnetId())
+                    .collect(Collectors.toList());
+            final List<Uuid> updateSnMapsIds = portupdateIps.stream().map(ip -> ip.getSubnetId())
+                    .collect(Collectors.toList());
+            for (Uuid snId: originalSnMapsIds) {
+                if (!updateSnMapsIds.remove(snId)) {
+                    // snId was present in originalSnMapsIds, but not in updateSnMapsIds
+                    nvpnManager.removePortsFromSubnetmapNode(snId, portoriginal.getUuid(), null);
+                }
             }
-            if (!subnetUpdated) {
-                Subnetmap subnetmap = NeutronvpnUtils.getSubnetmap(dataBroker, newSubnetId);
-                vpnIdNew = subnetmap != null ? subnetmap.getVpnId() : null;
+            for (Uuid snId: updateSnMapsIds) {
+                nvpnManager.updateSubnetmapNodeWithPorts(snId, portupdate.getUuid(), null);
             }
-            if (vpnIdNew != null) {
-                // remove vpn-interface for this neutron port
-                LOG.debug("removing VPN Interface for port {}", portoriginal.getUuid().getValue());
-                nvpnManager.deleteVpnInterface(vpnIdOld, null, portoriginal, wrtConfigTxn);
-                // create vpn-interface on this neutron port
-                LOG.debug("Adding VPN Interface for port {}", portupdate.getUuid().getValue());
-                nvpnManager.createVpnInterface(vpnIdNew, null, portupdate, wrtConfigTxn);
+            final Uuid oldVpnId = NeutronvpnUtils.getVpnForNetwork(dataBroker, portoriginal.getNetworkId());
+            if (oldVpnId != null) {
+                LOG.info("removing VPN Interface for port {}", portoriginal.getUuid().getValue());
+                nvpnManager.deleteVpnInterface(oldVpnId, portoriginal, wrtConfigTxn);
+                futures.add(wrtConfigTxn.submit());
+            }
+            final Uuid newVpnId = NeutronvpnUtils.getVpnForNetwork(dataBroker, portupdate.getNetworkId());
+            if (newVpnId != null) {
+                LOG.info("Adding VPN Interface for port {}", portupdate.getUuid().getValue());
+                nvpnManager.createVpnInterface(newVpnId, portupdate, wrtConfigTxn);
+                futures.add(wrtConfigTxn.submit());
             }
-            List<ListenableFuture<Void>> futures = new ArrayList<>();
-            futures.add(wrtConfigTxn.submit());
             return futures;
         });
     }
@@ -505,7 +532,6 @@ public class NeutronPortChangeListener extends AsyncDataTreeChangeListenerBase<P
         Interface inf = createInterface(port);
         String infName = inf.getName();
 
-        LOG.debug("Creating OFPort Interface {}", infName);
         InstanceIdentifier interfaceIdentifier = NeutronvpnUtils.buildVlanInterfaceIdentifier(infName);
         try {
             Optional<Interface> optionalInf = NeutronvpnUtils.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
index 281377d91ce9a9ca7c9bd4b439dd3e0cf046ebad..7d80da0ac4e8db681ff15a2b2be7d346c4cb7297 100644 (file)
@@ -23,6 +23,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 
@@ -164,6 +165,7 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
     private final IVpnManager vpnManager;
     private final NeutronEvpnManager neutronEvpnManager;
     private final NeutronEvpnUtils neutronEvpnUtils;
+    private static ConcurrentHashMap<Uuid, Uuid> unprocessedPortsMap = new ConcurrentHashMap<>();
 
     @Inject
     public NeutronvpnManager(
@@ -214,21 +216,32 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
         try {
             InstanceIdentifier<Subnetmap> subnetMapIdentifier = NeutronvpnUtils.buildSubnetMapIdentifier(subnetId);
             synchronized (subnetId.getValue().intern()) {
+                LOG.info("createSubnetmapNode: subnet ID {}", subnetId.toString());
                 Optional<Subnetmap> sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
                         LogicalDatastoreType.CONFIGURATION, subnetMapIdentifier);
                 if (sn.isPresent()) {
-                    LOG.error("subnetmap node for subnet ID {} already exists, returning", subnetId.getValue());
+                    LOG.error("createSubnetmapNode: Subnetmap node for subnet ID {} already exists, returning",
+                        subnetId.getValue());
                     return;
                 }
                 SubnetmapBuilder subnetmapBuilder = new SubnetmapBuilder().setKey(new SubnetmapKey(subnetId))
                         .setId(subnetId).setSubnetIp(subnetIp).setTenantId(tenantId).setNetworkId(networkId)
                         .setNetworkType(networkType).setSegmentationId(segmentationId);
-                LOG.debug("Adding a new subnet node in Subnetmaps DS for subnet {}", subnetId.getValue());
+                LOG.debug("createSubnetmapNode: Adding a new subnet node in Subnetmaps DS for subnet {}",
+                    subnetId.getValue());
                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
                         subnetMapIdentifier, subnetmapBuilder.build());
             }
         } catch (Exception e) {
-            LOG.error("Creating subnetmap node failed for subnet {}", subnetId.getValue());
+            LOG.error("createSubnetmapNode: Creating subnetmap node failed for subnet {}", subnetId.getValue());
+        }
+        // check if there are ports to update for already created Subnetmap node
+        LOG.debug("createSubnetmapNode: Update created Subnetmap for subnet {} with ports", subnetId.getValue());
+        for (Map.Entry<Uuid, Uuid> entry : unprocessedPortsMap.entrySet()) {
+            if (entry.getValue().getValue().equals(subnetId.getValue())) {
+                updateSubnetmapNodeWithPorts(subnetId, entry.getKey(), null);
+                unprocessedPortsMap.remove(entry.getKey());
+            }
         }
     }
 
@@ -308,6 +321,7 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
         Subnetmap subnetmap = null;
         InstanceIdentifier<Subnetmap> id = InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
                 new SubnetmapKey(subnetId)).build();
+        LOG.info("updateSubnetmapNodeWithPorts : subnetId {}, subnetMapId {}", subnetId.toString(), id.toString());
         try {
             synchronized (subnetId.getValue().intern()) {
                 Optional<Subnetmap> sn = NeutronvpnUtils.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
@@ -320,8 +334,8 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
                         }
                         portList.add(portId);
                         builder.setPortList(portList);
-                        LOG.debug("Updating existing subnetmap node {} with port {}", subnetId.getValue(),
-                                portId.getValue());
+                        LOG.debug("updateSubnetmapNodeWithPorts: Updating existing subnetmap node {} with port {}",
+                            subnetId.getValue(), portId.getValue());
                     }
                     if (null != directPortId) {
                         List<Uuid> directPortList = builder.getDirectPortList();
@@ -337,7 +351,9 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, id,
                             subnetmap);
                 } else {
-                    LOG.error("Trying to update non-existing subnetmap node {} ", subnetId.getValue());
+                    LOG.info("updateSubnetmapNodeWithPorts: Subnetmap node is not ready {}, put port {} in unprocessed "
+                        + "cache ", subnetId.getValue(), portId.getValue());
+                    unprocessedPortsMap.put(portId, subnetId);
                 }
             }
         } catch (Exception e) {
@@ -686,8 +702,7 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
         }
     }
 
-    protected void createVpnInterface(Uuid vpnId, Uuid routerId, Port port,
-                                      WriteTransaction wrtConfigTxn) {
+    protected void createVpnInterface(Uuid vpnId, Port port, WriteTransaction wrtConfigTxn) {
         String infName = port.getUuid().getValue();
         List<Adjacency> adjList = new ArrayList<>();
         Boolean isRouterInterface = false;
@@ -695,42 +710,41 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
             isRouterInterface = port.getDeviceOwner().equals(NeutronConstants.DEVICE_OWNER_ROUTER_INF);
         }
         LOG.trace("createVpnInterface - isRouterInterface:{}", isRouterInterface);
-        Router rtr = null;
-        if (routerId != null) {
-            rtr = NeutronvpnUtils.getNeutronRouter(dataBroker, routerId);
-        }
-        List<FixedIps> ips = port.getFixedIps();
         // create adjacency list
-        for (FixedIps ip : ips) {
-            // create vm adjacency
+        Uuid rtrId = null;
+        for (FixedIps ip : port.getFixedIps()) {
             String ipValue = String.valueOf(ip.getIpAddress().getValue());
             String ipPrefix = ip.getIpAddress().getIpv4Address() != null ? ipValue + "/32" : ipValue + "/128";
             Adjacency vmAdj = new AdjacencyBuilder().setKey(new AdjacencyKey(ipPrefix)).setIpAddress(ipPrefix)
                     .setMacAddress(port.getMacAddress().getValue()).setAdjacencyType(AdjacencyType.PrimaryAdjacency)
                     .setSubnetId(ip.getSubnetId()).build();
             adjList.add(vmAdj);
-            // create extra route adjacency
-            if (rtr != null && rtr.getRoutes() != null) {
-                List<Routes> routeList = rtr.getRoutes();
-                List<Adjacency> erAdjList = getAdjacencyforExtraRoute(vpnId, routeList, ipValue);
-                if (erAdjList != null && !erAdjList.isEmpty()) {
-                    adjList.addAll(erAdjList);
+
+            NeutronvpnUtils.createVpnPortFixedIpToPort(dataBroker, vpnId.getValue(), ipValue, infName,
+                    port.getMacAddress().getValue(), isRouterInterface, wrtConfigTxn);
+
+            Uuid routerId = NeutronvpnUtils.getSubnetmap(dataBroker, ip.getSubnetId()).getRouterId();
+            if (routerId != null && routerId != rtrId) {
+                addToNeutronRouterInterfacesMap(routerId, infName);
+                Router rtr = NeutronvpnUtils.getNeutronRouter(dataBroker, routerId);
+                if (rtr != null && rtr.getRoutes() != null) {
+                    List<Routes> routeList = rtr.getRoutes();
+                    List<Adjacency> erAdjList = getAdjacencyforExtraRoute(vpnId, routeList, ipValue);
+                    if (erAdjList != null && !erAdjList.isEmpty()) {
+                        adjList.addAll(erAdjList);
+                    }
                 }
+                rtrId = routerId;
             }
-            NeutronvpnUtils.createVpnPortFixedIpToPort(dataBroker, vpnId.getValue(), ipValue, infName, port
-                            .getMacAddress().getValue(), isRouterInterface, wrtConfigTxn);
         }
         // create vpn-interface on this neutron port
         Adjacencies adjs = new AdjacenciesBuilder().setAdjacency(adjList).build();
         writeVpnInterfaceToDs(vpnId, infName, adjs, isRouterInterface, wrtConfigTxn);
-        if (routerId != null) {
-            addToNeutronRouterInterfacesMap(routerId, infName);
-        }
     }
 
     // TODO Clean up the exception handling
     @SuppressWarnings("checkstyle:IllegalCatch")
-    protected void deleteVpnInterface(Uuid vpnId, Uuid routerId, Port port, WriteTransaction wrtConfigTxn) {
+    protected void deleteVpnInterface(Uuid vpnId, Port port, WriteTransaction wrtConfigTxn) {
         Boolean wrtConfigTxnPresent = true;
         if (wrtConfigTxn == null) {
             wrtConfigTxnPresent = false;
@@ -742,18 +756,21 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
             LOG.debug("Deleting vpn interface {}", infName);
             wrtConfigTxn.delete(LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 
+            Uuid rtrId = null;
             List<FixedIps> ips = port.getFixedIps();
             for (FixedIps ip : ips) {
                 String ipValue = String.valueOf(ip.getIpAddress().getValue());
                 NeutronvpnUtils.removeVpnPortFixedIpToPort(dataBroker, vpnId.getValue(),
                         ipValue, wrtConfigTxn);
+                Uuid routerId = NeutronvpnUtils.getSubnetmap(dataBroker, ip.getSubnetId()).getRouterId();
+                if (routerId != null && routerId != rtrId) {
+                    removeFromNeutronRouterInterfacesMap(routerId, infName);
+                    rtrId = routerId;
+                }
             }
         } catch (Exception ex) {
             LOG.error("Deletion of vpninterface {} failed", infName, ex);
         }
-        if (routerId != null) {
-            removeFromNeutronRouterInterfacesMap(routerId, infName);
-        }
         if (!wrtConfigTxnPresent) {
             wrtConfigTxn.submit();
         }
@@ -869,7 +886,8 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
      * @throws Exception if association of L3VPN failed
      */
     public void createVpn(Uuid vpn, String name, Uuid tenant, List<String> rd, List<String> irt, List<String> ert,
-                            Uuid router, List<Uuid> networks, VpnInstance.Type type, long l3vni) throws Exception {
+                            Uuid router, List<Uuid> networks, VpnInstance.Type type, long l3vni)
+                                    throws Exception {
 
         // Update VPN Instance node
         updateVpnInstanceNode(vpn.getValue(), rd, irt, ert, type, l3vni);
@@ -958,8 +976,8 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
                 Uuid vpnId = NeutronvpnUtils.getVpnForRouter(dataBroker, vpn.getRouterId(), true);
                 if (vpnId != null) {
                     msg = String.format("Creation of L3VPN failed for VPN %s due to router %s already associated to "
-                                    + "another VPN %s", vpn.getId().getValue(), vpn.getRouterId().getValue(),
-                            vpnId.getValue());
+                            + "another VPN %s", vpn.getId().getValue(), vpn.getRouterId().getValue(),
+                                vpnId.getValue());
                     LOG.warn(msg);
                     error = RpcResultBuilder.newWarning(ErrorType.PROTOCOL, "invalid-input", msg);
                     errorList.add(error);
@@ -1114,7 +1132,7 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
                 if (optionalVpnMap.isPresent()) {
                     VpnMap vpnMap = optionalVpnMap.get();
                     l3vpn.setRouterId(vpnMap.getRouterId()).setNetworkIds(vpnMap.getNetworkIds())
-                            .setTenantId(vpnMap.getTenantId()).setName(vpnMap.getName());
+                        .setTenantId(vpnMap.getTenantId()).setName(vpnMap.getName());
                 }
                 l3vpnList.add(l3vpn.build());
             }
@@ -1215,7 +1233,6 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
             return;
         }
 
-        final Uuid routerId = NeutronvpnUtils.getVpnMap(dataBroker, vpnId).getRouterId();
         final VpnInstance vpnInstance = VpnHelper.getVpnInstance(dataBroker, vpnId.getValue());
         if (isVpnOfTypeL2(vpnInstance)) {
             neutronEvpnUtils.updateElanAndVpn(vpnInstance, sn.getNetworkId().getValue(),
@@ -1225,14 +1242,13 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
         // vpn-interfaces
         List<Uuid> portList = sn.getPortList();
         if (portList != null) {
-            for (final Uuid portId : sn.getPortList()) {
+            for (final Uuid portId : portList) {
                 LOG.debug("adding vpn-interface for port {}", portId.getValue());
                 final DataStoreJobCoordinator portDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
                 portDataStoreCoordinator.enqueueJob("PORT-" + portId.getValue(), () -> {
                     WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
                     List<ListenableFuture<Void>> futures = new ArrayList<>();
-                    createVpnInterface(vpnId, routerId, NeutronvpnUtils.getNeutronPort(dataBroker, portId),
-                            wrtConfigTxn);
+                    createVpnInterface(vpnId, NeutronvpnUtils.getNeutronPort(dataBroker, portId), wrtConfigTxn);
                     futures.add(wrtConfigTxn.submit());
                     return futures;
                 });
@@ -1593,8 +1609,6 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
                     vpnId.getValue(), subnet.getValue());
             return;
         }
-
-        final Uuid routerId = vpnMap.getRouterId();
         Subnetmap sn = NeutronvpnUtils.getSubnetmap(dataBroker, subnet);
         final VpnInstance vpnInstance = VpnHelper.getVpnInstance(dataBroker, vpnId.getValue());
         if (isVpnOfTypeL2(vpnInstance)) {
@@ -1605,15 +1619,16 @@ public class NeutronvpnManager implements NeutronvpnService, AutoCloseable, Even
             // Check if there are ports on this subnet; remove corresponding vpn-interfaces
             List<Uuid> portList = sn.getPortList();
             if (portList != null) {
-                for (final Uuid portId : sn.getPortList()) {
+                for (final Uuid portId : portList) {
                     LOG.debug("removing vpn-interface for port {}", portId.getValue());
                     final DataStoreJobCoordinator portDataStoreCoordinator = DataStoreJobCoordinator.getInstance();
                     portDataStoreCoordinator.enqueueJob("PORT-" + portId.getValue(), () -> {
                         WriteTransaction wrtConfigTxn = dataBroker.newWriteOnlyTransaction();
                         List<ListenableFuture<Void>> futures = new ArrayList<>();
                         Port port = NeutronvpnUtils.getNeutronPort(dataBroker, portId);
+
                         if (port != null) {
-                            deleteVpnInterface(vpnId, routerId, port, wrtConfigTxn);
+                            deleteVpnInterface(vpnId, port, wrtConfigTxn);
                         } else {
                             LOG.error("Cannot proceed with deleteVpnInterface for port {} in subnet {} since port is "
                                 + "absent in Neutron config DS", portId.getValue(), subnet.getValue());