Bug 4277 - Deleted Network flow entries retained in br-int, if network associated...
[ovsdb.git] / openstack / net-virt / src / main / java / org / opendaylight / ovsdb / openstack / netvirt / impl / NeutronL3Adapter.java
index 1c58e6ca6062aeb7bbd1bedc9f44306926e76e82..603ecc58315a1fdc04fe25b437c08556ee779690 100644 (file)
@@ -1,12 +1,11 @@
 /*
- * Copyright (C) 2014 Red Hat, Inc.
+ * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
  * and is available at http://www.eclipse.org/legal/epl-v10.html
- *
- * Authors : Dave Tucker, Flavio Fernandes
  */
+
 package org.opendaylight.ovsdb.openstack.netvirt.impl;
 
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -57,7 +56,7 @@ import java.util.concurrent.Executors;
  * as well as the multi-tenant router forwarding provider.
  */
 public class NeutronL3Adapter implements ConfigInterface {
-    private static final Logger LOGGER = LoggerFactory.getLogger(NeutronL3Adapter.class);
+    private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
 
     // The implementation for each of these services is resolved by the OSGi Service Manager
     private volatile ConfigurationService configurationService;
@@ -74,12 +73,17 @@ public class NeutronL3Adapter implements ConfigInterface {
     private volatile GatewayMacResolver gatewayMacResolver;
 
     private class FloatIpData {
-        private final Long dpid;          // br-int of node where floating ip is associated with tenant port
-        private final Long ofPort;        // patch port in br-int used to reach br-ex
-        private final String segId;       // segmentation id of the net where fixed ip is instantiated
-        private final String macAddress;  // mac address assigned to neutron port of floating ip
+        // br-int of node where floating ip is associated with tenant port
+        private final Long dpid;
+        // patch port in br-int used to reach br-ex
+        private final Long ofPort;
+        // segmentation id of the net where fixed ip is instantiated
+        private final String segId;
+        // mac address assigned to neutron port of floating ip
+        private final String macAddress;
         private final String floatingIpAddress;
-        private final String fixedIpAddress;  // ip address given to tenant vm
+        // ip address given to tenant vm
+        private final String fixedIpAddress;
         private final String neutronRouterMac;
 
         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
@@ -118,7 +122,7 @@ public class NeutronL3Adapter implements ConfigInterface {
     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 
     public NeutronL3Adapter() {
-        LOGGER.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
+        LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
     }
 
     private void initL3AdapterMembers() {
@@ -143,15 +147,15 @@ public class NeutronL3Adapter implements ConfigInterface {
             }
 
             this.enabled = true;
-            LOGGER.info("OVSDB L3 forwarding is enabled");
+            LOG.info("OVSDB L3 forwarding is enabled");
             if (configurationService.isDistributedArpDisabled()) {
                 this.flgDistributedARPEnabled = false;
-                LOGGER.info("Distributed ARP responder is disabled");
+                LOG.info("Distributed ARP responder is disabled");
             } else {
-                LOGGER.debug("Distributed ARP responder is enabled");
+                LOG.debug("Distributed ARP responder is enabled");
             }
         } else {
-            LOGGER.debug("OVSDB L3 forwarding is disabled");
+            LOG.debug("OVSDB L3 forwarding is disabled");
         }
     }
 
@@ -181,10 +185,7 @@ public class NeutronL3Adapter implements ConfigInterface {
      * @param subnet An instance of NeutronSubnet object.
      */
     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
-        LOGGER.debug("Neutron subnet {} event : {}", action, subnet.toString());
-        if (!this.enabled) {
-            return;
-        }
+        LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
     }
 
     /**
@@ -196,7 +197,7 @@ public class NeutronL3Adapter implements ConfigInterface {
      * @param neutronPort An instance of NeutronPort object.
      */
     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
-        LOGGER.debug("Neutron port {} event : {}", action, neutronPort.toString());
+        LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
         if (!this.enabled) {
             return;
         }
@@ -207,21 +208,21 @@ public class NeutronL3Adapter implements ConfigInterface {
             if(!isDelete){
                 Node externalBridgeNode = getExternalBridgeNode();
                 if(externalBridgeNode != null){
-                    LOGGER.info("Port {} is network router gateway interface, "
-                            + "triggering gateway resolution for the attached external network on node {}",neutronPort,externalBridgeNode);
+                    LOG.info("Port {} is network router gateway interface, "
+                            + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
                 }else{
-                    LOGGER.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
+                    LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
                 }
             }else{
                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 
-                if(externalNetwork != null){
-                    if(externalNetwork.isRouterExternal()){
-                        final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
-                        if(externalSubnet != null){
-                            gatewayMacResolver.stopPeriodicReferesh(new Ipv4Address(externalSubnet.getGatewayIP()));
-                        }
+                if (externalNetwork != null && externalNetwork.isRouterExternal()) {
+                    final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
+                    // TODO support IPv6
+                    if (externalSubnet != null &&
+                            externalSubnet.getIpVersion() == 4) {
+                        gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
                     }
                 }
             }
@@ -236,7 +237,8 @@ public class NeutronL3Adapter implements ConfigInterface {
             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
                 NeutronRouter_Interface neutronRouterInterface =
                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
-                neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
+                // id of router interface to be same as subnet
+                neutronRouterInterface.setID(neutronIP.getSubnetUUID());
                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 
                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
@@ -247,7 +249,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             // need to do this check here because a router interface is not added to a node until tenant becomes needed
             // there.
             //
-            if (!isDelete) {
+            if (!isDelete && neutronPort.getFixedIPs() != null) {
                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
                     NeutronRouter_Interface neutronRouterInterface =
                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
@@ -267,10 +269,7 @@ public class NeutronL3Adapter implements ConfigInterface {
      * @param neutronRouter An instance of NeutronRouter object.
      */
     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
-        LOGGER.debug("Neutron router {} event : {}", action, neutronRouter.toString());
-        if (!this.enabled) {
-            return;
-        }
+        LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
     }
 
     /**
@@ -284,7 +283,7 @@ public class NeutronL3Adapter implements ConfigInterface {
     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
                                                   final NeutronRouter_Interface neutronRouterInterface,
                                                   Action action) {
-        LOGGER.debug("Router interface {} got event {}. Subnet {}",
+        LOG.debug("Router interface {} got event {}. Subnet {}",
                      neutronRouterInterface.getPortUUID(),
                      action,
                      neutronRouterInterface.getSubnetUUID());
@@ -312,6 +311,13 @@ public class NeutronL3Adapter implements ConfigInterface {
             }
             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
         }
+
+        if (isDelete) {
+            /*
+             *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
+             */
+            this.cleanupRouterCache(neutronRouterInterface);
+        }
     }
 
     /**
@@ -327,7 +333,7 @@ public class NeutronL3Adapter implements ConfigInterface {
                                              Action actionIn) {
         Preconditions.checkNotNull(neutronFloatingIP);
 
-        LOGGER.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
+        LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
                 neutronFloatingIP.getFixedIPAddress(),
                 neutronFloatingIP.getFloatingIPAddress(),
                 neutronFloatingIP.getFloatingNetworkUUID());
@@ -348,7 +354,8 @@ public class NeutronL3Adapter implements ConfigInterface {
         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 
         if (action != Action.DELETE) {
-            programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
+            // must be first, as it updates floatIpDataMapCache
+            programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 
             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
@@ -356,7 +363,8 @@ public class NeutronL3Adapter implements ConfigInterface {
             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 
-            programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
+            // must be last, as it updates floatIpDataMapCache
+            programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
         }
     }
 
@@ -369,7 +377,7 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
         if (fid == null) {
-            LOGGER.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
+            LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
             return;
         }
@@ -386,7 +394,7 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
         if (fid == null) {
-            LOGGER.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
+            LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
             return;
         }
@@ -416,7 +424,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 
         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
-            LOGGER.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
+            LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
             return;
         }
@@ -441,7 +449,7 @@ public class NeutronL3Adapter implements ConfigInterface {
                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
                 floatingIpMac == null || floatingIpMac.isEmpty() ||
                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
-            LOGGER.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
+            LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
                     fixedIpAddress,
                     floatingIpAddress,
                     neutronPortForFloatIp,
@@ -456,7 +464,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         final Long dpId = nodeIfPair.getLeft();
         final Long ofPort = findOFPortForExtPatch(dpId);
         if (ofPort == null) {
-            LOGGER.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
+            LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
                     dpId);
             return;
         }
@@ -468,7 +476,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
-            LOGGER.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
+            LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
                     floatingIpMac, ofPort, providerSegmentationId, dpId);
         }
@@ -477,19 +485,19 @@ public class NeutronL3Adapter implements ConfigInterface {
     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
         if (floatIpData == null) {
-            LOGGER.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
+            LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
             return;
         }
 
         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
                 floatIpData.floatingIpAddress, Action.DELETE)) {
             floatIpDataMapCache.remove(neutronFloatingIPUuid);
-            LOGGER.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
+            LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
         }
     }
 
-    private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
+    private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
@@ -499,7 +507,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         return null;
     }
 
-    private final Long findOFPortForExtPatch(Long dpId) {
+    private Long findOFPortForExtPatch(Long dpId) {
         final String brInt = configurationService.getIntegrationBridgeName();
         final String brExt = configurationService.getExternalBridgeName();
         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
@@ -507,7 +515,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         Preconditions.checkNotNull(dpId);
         Preconditions.checkNotNull(portNameInt);
 
-        final long dpidPrimitive = dpId.longValue();
+        final long dpidPrimitive = dpId;
         for (Node node : nodeCacheManager.getBridgeNodes()) {
             if (dpidPrimitive == southbound.getDataPathId(node)) {
                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
@@ -525,10 +533,7 @@ public class NeutronL3Adapter implements ConfigInterface {
      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
      */
     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
-        LOGGER.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
-        if (!this.enabled) {
-            return;
-        }
+        LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
     }
 
     //
@@ -546,7 +551,7 @@ public class NeutronL3Adapter implements ConfigInterface {
      */
     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
                                      final NeutronNetwork neutronNetwork, Action action) {
-        LOGGER.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
+        LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
         if (!this.enabled) {
             return;
@@ -556,7 +561,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
         final Uuid interfaceUuid = intf.getInterfaceUuid();
 
-        LOGGER.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
+        LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 
         if (neutronPort != null) {
@@ -577,7 +582,7 @@ public class NeutronL3Adapter implements ConfigInterface {
 
     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
-        LOGGER.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
+        LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
                 neutronPortUuid, dpId, interfaceUuid.getValue());
     }
 
@@ -586,7 +591,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
             final String currPortUuid = entry.getKey();
             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
-                LOGGER.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
+                LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
                 neutronPortToDpIdCache.remove(currPortUuid);
                 break;
@@ -627,7 +632,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         final Action action = isDelete ? Action.DELETE : Action.ADD;
         List<Node> nodes = nodeCacheManager.getBridgeNodes();
         if (nodes.isEmpty()) {
-            LOGGER.trace("updateL3ForNeutronPort has no nodes to work with");
+            LOG.trace("updateL3ForNeutronPort has no nodes to work with");
         }
         for (Node node : nodes) {
             final Long dpid = getDpidForIntegrationBridge(node);
@@ -646,7 +651,7 @@ public class NeutronL3Adapter implements ConfigInterface {
                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 
                 // Configure distributed ARP responder
-                if (true == flgDistributedARPEnabled) {
+                if (flgDistributedARPEnabled) {
                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
                 }
             }
@@ -660,15 +665,15 @@ public class NeutronL3Adapter implements ConfigInterface {
         // will look at desired action for node.
 
         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
-        final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
+        final boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 
-        if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            LOGGER.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
+        if (actionForNode == Action.DELETE && !isProgrammed) {
+            LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
             return;
         }
-        if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
-            LOGGER.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
+        if (actionForNode == Action.ADD && isProgrammed) {
+            LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
             return;
         }
@@ -701,11 +706,11 @@ public class NeutronL3Adapter implements ConfigInterface {
         }
 
         if (status.isSuccess()) {
-            LOGGER.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
+            LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
                          l3ForwardingProvider == null ? "skipped" : "programmed",
                          macAddress, address, node.getNodeId().getValue(), actionForNode);
         } else {
-            LOGGER.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
+            LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
         }
         return status;
@@ -725,12 +730,11 @@ public class NeutronL3Adapter implements ConfigInterface {
                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
         final String destinationSegmentationId = neutronNetwork != null ?
                                                  neutronNetwork.getProviderSegmentationID() : null;
-        final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
         final String cidr = subnet != null ? subnet.getCidr() : null;
         final int mask = getMaskLenFromCidr(cidr);
 
-        LOGGER.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
+        LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
                      destNeutronRouterInterface, isDelete);
 
         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
@@ -748,7 +752,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             cidr == null || cidr.isEmpty() ||
             macAddress == null || macAddress.isEmpty() ||
             ipList == null || ipList.isEmpty()) {
-            LOGGER.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
+            LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
                          destinationSegmentationId, cidr, macAddress, ipList);
             // done: go no further w/out all the info needed...
             return;
@@ -766,7 +770,7 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         List<Node> nodes = nodeCacheManager.getBridgeNodes();
         if (nodes.isEmpty()) {
-            LOGGER.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
+            LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
         }
         for (Node node : nodes) {
             final Long dpid = getDpidForIntegrationBridge(node);
@@ -777,7 +781,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             for (Neutron_IPs neutronIP : ipList) {
                 final String ipStr = neutronIP.getIpAddress();
                 if (ipStr.isEmpty()) {
-                    LOGGER.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
+                    LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
                             node.getNodeId().getValue(), ipStr);
                     continue;
                 }
@@ -809,13 +813,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             }
         }
 
-        // Keep cache for finding router's mac from network uuid -- remove
-        //
-        if (isDelete) {
-            networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
-            networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
-            subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
-        }
+        // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
     }
 
     private void programFlowForNetworkFromExternal(final Node node,
@@ -845,7 +843,7 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
         if (sourceSubnetId == null) {
-            LOGGER.error("Could not get provider Subnet ID from router interface {}",
+            LOG.error("Could not get provider Subnet ID from router interface {}",
                          srcNeutronRouterInterface.getID());
             return;
         }
@@ -853,13 +851,13 @@ public class NeutronL3Adapter implements ConfigInterface {
         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
         if (sourceNetworkId == null) {
-            LOGGER.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
+            LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
             return;
         }
 
         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
         if (sourceNetwork == null) {
-            LOGGER.error("Could not get provider Network for Network ID {}", sourceNetworkId);
+            LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
             return;
         }
 
@@ -869,7 +867,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         }
         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
         if (sourceSegmentationId == null) {
-            LOGGER.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
+            LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
             return;
         }
         if (sourceSegmentationId.equals(destinationSegmentationId)) {
@@ -891,7 +889,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             if (cidr2 == null || cidr2.isEmpty() ||
                 macAddress2 == null || macAddress2.isEmpty() ||
                 ipList2 == null || ipList2.isEmpty()) {
-                LOGGER.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
+                LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
                              sourceSegmentationId, cidr2, macAddress2, ipList2);
                 // done: go no further w/out all the info needed...
                 return;
@@ -921,17 +919,17 @@ public class NeutronL3Adapter implements ConfigInterface {
         final String cacheKey = node.getNodeId().getValue() + ":" +
                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
                                 ipStr + "/" + Integer.toString(mask);
-        final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
+        final boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 
-        if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            LOGGER.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
+        if (actionForNode == Action.DELETE && !isProgrammed) {
+            LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
                          " action {} is already done",
                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
                          macAddress, ipStr, mask, actionForNode);
             return;
         }
-        if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
-            LOGGER.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
+        if (actionForNode == Action.ADD && isProgrammed) {
+            LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
                          " action {} is already done",
                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
                          macAddress, ipStr, mask, actionForNode);
@@ -969,12 +967,12 @@ public class NeutronL3Adapter implements ConfigInterface {
         }
 
         if (status.isSuccess()) {
-            LOGGER.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
+            LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
                          routingProvider == null ? "skipped" : "programmed",
                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
                          actionForNode);
         } else {
-            LOGGER.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
+            LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
                          actionForNode, status);
         }
@@ -988,15 +986,15 @@ public class NeutronL3Adapter implements ConfigInterface {
         // will look at desired action for node.
         //
         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
-        final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
+        final boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 
-        if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            LOGGER.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
+        if (action == Action.DELETE && !isProgrammed) {
+            LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
                     dpid, segOrOfPort, macAddress, ipStr, action);
             return true;
         }
-        if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
-            LOGGER.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
+        if (action == Action.ADD && isProgrammed) {
+            LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
                     dpid, segOrOfPort, macAddress, ipStr, action);
             return true;
         }
@@ -1031,11 +1029,11 @@ public class NeutronL3Adapter implements ConfigInterface {
         }
 
         if (status.isSuccess()) {
-            LOGGER.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
+            LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
                          arpProvider == null ? "skipped" : "programmed",
                          macAddress, address, dpid, segOrOfPort, action);
         } else {
-            LOGGER.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
+            LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
                          macAddress, address, dpid, segOrOfPort, action, status);
         }
         return status;
@@ -1048,16 +1046,16 @@ public class NeutronL3Adapter implements ConfigInterface {
         // will look at desired action for node.
         //
         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
-        final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
+        final boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
 
-        if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            LOGGER.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
+        if (action == Action.DELETE && !isProgrammed) {
+            LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
                     " action {} is already done",
                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
             return true;
         }
-        if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
-            LOGGER.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
+        if (action == Action.ADD && isProgrammed) {
+            LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
                     " action is already done",
                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
             return true;
@@ -1095,11 +1093,11 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         if (status.isSuccess()) {
             final boolean isSkipped = inboundNatProvider == null;
-            LOGGER.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
+            LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
                     isSkipped ? "skipped" : "programmed",
                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
         } else {
-            LOGGER.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
+            LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
                          " status:{}",
                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
                     status);
@@ -1113,15 +1111,15 @@ public class NeutronL3Adapter implements ConfigInterface {
         // will look at desired action for node.
         //
         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
-        final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
+        final boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
 
-        if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            LOGGER.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
+        if (actionForRewriteExclusion == Action.DELETE && !isProgrammed) {
+            LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
             return;
         }
-        if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
-            LOGGER.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
+        if (actionForRewriteExclusion == Action.ADD && isProgrammed) {
+            LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
             return;
         }
@@ -1145,11 +1143,11 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         if (status.isSuccess()) {
             final boolean isSkipped = outboundNatProvider == null;
-            LOGGER.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
+            LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
                          isSkipped ? "skipped" : "programmed",
                          cidr, node.getNodeId().getValue(), actionForNode);
         } else {
-            LOGGER.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
+            LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
                          cidr, node.getNodeId().getValue(), actionForNode, status);
         }
         return status;
@@ -1160,16 +1158,16 @@ public class NeutronL3Adapter implements ConfigInterface {
         // will look at desired action for node.
         //
         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
-        final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
+        final boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
 
-        if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            LOGGER.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
+        if (action == Action.DELETE && !isProgrammed) {
+            LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
                          "is already done",
                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
             return;
         }
-        if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
-            LOGGER.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
+        if (action == Action.ADD && isProgrammed) {
+            LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
                          "is already done",
                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
             return;
@@ -1202,12 +1200,12 @@ public class NeutronL3Adapter implements ConfigInterface {
 
         if (status.isSuccess()) {
             final boolean isSkipped = outboundNatProvider == null;
-            LOGGER.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
+            LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
                             " action {}",
                          isSkipped ? "skipped" : "programmed",
                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
         } else {
-            LOGGER.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
+            LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
                          " action {} status:{}",
                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
         }
@@ -1263,16 +1261,35 @@ public class NeutronL3Adapter implements ConfigInterface {
     }
 
     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
-        NeutronSubnet extSubnet = null;
-        for (NeutronSubnet subnet : neutronSubnetCache.getAllSubnets()){
-            if(subnet.getPortsInSubnet().contains(gatewayPort)){
-                extSubnet = subnet;
-                break;
+        for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
+            String subnetUUID = neutronIPs.getSubnetUUID();
+            NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
+            if (extSubnet != null && extSubnet.getGatewayIP() != null) {
+                return extSubnet;
             }
+            if (extSubnet == null) {
+                // TODO: when subnet is created, try again.
+                LOG.debug("subnet {} in not found", subnetUUID);
+             }
         }
-        return extSubnet;
+        return null;
     }
 
+     private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
+         /*
+          *  Fix for 4277
+          *  Remove the router cache only after deleting the neutron
+          *  port l3 flows.
+          */
+         final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
+
+         if (neutronPort != null) {
+             networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
+             networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
+             subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
+         }
+     }
+
     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
 
         Preconditions.checkNotNull(node);
@@ -1282,43 +1299,43 @@ public class NeutronL3Adapter implements ConfigInterface {
         if(externalNetwork != null){
             if(externalNetwork.isRouterExternal()){
                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
-                if(externalSubnet != null){
-                    if(externalSubnet.getGatewayIP() != null){
-                        LOGGER.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
-
-                        ListenableFuture<MacAddress> gatewayMacAddress =
-                                gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
-                                        new Ipv4Address(externalSubnet.getGatewayIP()),
-                                        new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
-                                        new MacAddress(gatewayPort.getMacAddress()),
-                                        false);
-                        if(gatewayMacAddress != null){
-                            Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
-                                @Override
-                                public void onSuccess(MacAddress result) {
-                                    if(result != null){
+
+                // TODO: address IPv6 case.
+                if (externalSubnet != null &&
+                    externalSubnet.getIpVersion() == 4) {
+                    LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
+                    ListenableFuture<MacAddress> gatewayMacAddress =
+                        gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
+                                                             new Ipv4Address(externalSubnet.getGatewayIP()),
+                                                             new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
+                                                             new MacAddress(gatewayPort.getMacAddress()),
+                                                             true);
+                    if(gatewayMacAddress != null){
+                        Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
+                            @Override
+                            public void onSuccess(MacAddress result) {
+                                if(result != null){
+                                    if(!result.getValue().equals(externalRouterMac)){
                                         updateExternalRouterMac(result.getValue());
-                                        LOGGER.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
-                                    }else{
-                                        LOGGER.warn("MAC address resolution failed for gateway IP {}",externalSubnet.getGatewayIP());
+                                        LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
                                     }
+                                }else{
+                                    LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
                                 }
+                            }
 
-                                @Override
-                                public void onFailure(Throwable t) {
-                                    LOGGER.warn("MAC address resolution failed for gateway IP {}",externalSubnet.getGatewayIP());
-                                }
-                            }, gatewayMacResolverPool);
-                        }
-                    }else{
-                        LOGGER.warn("No gateway IP address found for external subnet {}",externalSubnet);
+                            @Override
+                            public void onFailure(Throwable t) {
+                                LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
+                            }
+                        }, gatewayMacResolverPool);
                     }
-                }else{
-                    LOGGER.warn("Neutron subnet not found for external network {}",externalNetwork);
+                } else {
+                    LOG.warn("No gateway IP address found for external network {}", externalNetwork);
                 }
             }
         }else{
-            LOGGER.warn("Neutron network not found for router interface {}",gatewayPort);
+            LOG.warn("Neutron network not found for router interface {}", gatewayPort);
         }
     }