Bug 8937 : High CPU utilization of Java process due to SNAT packet looping

Description:
If we have 2 VMs with same IP belong to two different network and the
subnet associated with different routers having same external network and
external bgpvpn. In such as case, when traffic initiated using same
internal port some both the VMs, one of the VMs doesn't undergo SNAT
translation and the packet gets continously looped between Controller and OVS.

When the packet is punted to controller, check is only done if the packet
is already recevied for an given internal-ip and internal-port. As a
result, VM1 packet is processed properly and SNAT translation happens. But
when VM2 packet arrives, we see with internal-ip and internal-port already
processed and hence we sent back to OVS without SNAT translation which
resulted in looping.

Change done to have a check of router-id along with internal-ip and
internal-port to uniquely differentiate VM1 and VM2 traffic seperatly and
perform SNAT translation properly for both.

Change-Id: I29b6aef96f353bef711f3fb2c1f0b9f6d75507a6
Signed-off-by: cgowdru <chetan.arakere@altencalsoftlabs.com>

diff --git a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java
index 77acf777fd3d5dd6202a09b8c117ed418e8f3908..ef78dc3e05ea84e34db24ef15cc4290975c8c290 100644 (file)
--- a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java
+++ b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java
@@ -1367,8 +1367,10 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                             // which were containing the removed external IP
                             naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort, protocolType);
                             //Remove the IP port incomint packer map.
-                            naptPacketInHandler.removeIncomingPacketMap(removedInternalIpPort);
-                            String[] removedInternalIpPortParts = removedInternalIpPort.split(":");
+                            naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR
+                                    + removedInternalIpPort);
+                            String[] removedInternalIpPortParts = removedInternalIpPort
+                                    .split(NatConstants.COLON_SEPARATOR);
                             if (removedInternalIpPortParts.length == 2) {
                                 String removedInternalIp = removedInternalIpPortParts[0];
                                 String removedInternalPort = removedInternalIpPortParts[1];
@@ -1411,6 +1413,8 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                         List<String> internalPorts = internalIpPort.getValue();
                         for (String internalPort : internalPorts) {
                             //Remove the NAPT translation entries from Outbound NAPT table
+                            naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR
+                                    + internalIp + NatConstants.COLON_SEPARATOR + internalPort);
                             naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
                                     routerId, internalIp, Integer.valueOf(internalPort));
                         }
@@ -1904,6 +1908,8 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                 String internalPort = ipPortParts[1];
 
                 //Build the flow for the outbound NAPT table
+                naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
+                        + NatConstants.COLON_SEPARATOR + internalPort);
                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
                     String.valueOf(routerId), internalIp, Integer.valueOf(internalPort));
                 FlowEntity outboundNaptFlowEntity =
@@ -2015,6 +2021,8 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                     String internalPort = ipPortParts[1];
 
                     //Build the flow for the outbound NAPT table
+                    naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
+                            + NatConstants.COLON_SEPARATOR + internalPort);
                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
                         String.valueOf(routerId), internalIp, Integer.valueOf(internalPort));
                     FlowEntity outboundNaptFlowEntity =

diff --git a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptEventHandler.java b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptEventHandler.java
index bc3e4503b8353af16ea8c38f1267fd28bf7f27ba..5114622678e858358911d84a3b7d35dcc35c68ee 100644 (file)
--- a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptEventHandler.java
+++ b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptEventHandler.java
@@ -281,7 +281,9 @@ public class NaptEventHandler {
                                             internalAddress, externalAddress);
                                 }
                             });
-                    String key = naptEntryEvent.getIpAddress() + ":" + naptEntryEvent.getPortNumber();
+                    String key = naptEntryEvent.getRouterId() + NatConstants.COLON_SEPARATOR
+                            + naptEntryEvent.getIpAddress() + NatConstants.COLON_SEPARATOR
+                            + naptEntryEvent.getPortNumber();
                     NatPacketProcessingState state = NaptPacketInHandler.INCOMING_PACKET_MAP.get(key);
                     state.setFlowInstalledTime(System.currentTimeMillis());
                 } else {

diff --git a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java
index 01b32ad5bed42c6d6246a1751ac75d2d082160a0..4d908aaf6dabed64224ac098ca886b295e4a7649 100644 (file)
--- a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java
+++ b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java
@@ -145,7 +145,8 @@ public class NaptFlowRemovedEventHandler implements SalFlowListener {
                 LOG.error("onFlowRemoved : Null exception while retrieving routerId");
                 return;
             }
-
+            final String internalIpPortKey = routerId + NatConstants.COLON_SEPARATOR
+                    + internalIpv4HostAddress + NatConstants.COLON_SEPARATOR + internalPortNumber;
             //Get the external IP address and the port from the model
             IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId,
                 internalIpv4HostAddress, internalPortNumber.toString(), protocol);
@@ -192,7 +193,6 @@ public class NaptFlowRemovedEventHandler implements SalFlowListener {
             FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef);
             long startTime = System.currentTimeMillis();
             mdsalManager.removeFlow(snatFlowEntity);
-            String internalIpPortKey = internalIpv4HostAddress + ":" + internalPortNumber;
             LOG.debug("onFlowRemoved : Elapsed time fo deleting table-{} flow for snat ({}) session:{}ms",
                     tableId, internalIpPortKey, (System.currentTimeMillis() - startTime));
             //Remove the SourceIP:Port key from the Napt packet handler map.

diff --git a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptPacketInHandler.java b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptPacketInHandler.java
index 1ffc44741a935cfb781228f9f32120c9d8ca6184..403c53070474ef7e7b0350854e7dca0b6f517dd3 100644 (file)
--- a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptPacketInHandler.java
+++ b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptPacketInHandler.java
@@ -122,7 +122,8 @@ public class NaptPacketInHandler implements PacketProcessingListener {
                         LOG.error("onPacketReceived : Router ID is invalid");
                         return;
                     }
-                    String sourceIPPortKey = internalIPAddress + ":" + portNumber;
+                    String sourceIPPortKey = routerId + NatConstants.COLON_SEPARATOR
+                            + internalIPAddress + NatConstants.COLON_SEPARATOR + portNumber;
                     if (!INCOMING_PACKET_MAP.containsKey(sourceIPPortKey)) {
                         INCOMING_PACKET_MAP.put(sourceIPPortKey,
                                 new NatPacketProcessingState(System.currentTimeMillis(), -1));
@@ -144,8 +145,7 @@ public class NaptPacketInHandler implements PacketProcessingListener {
                             NatPacketProcessingState state = INCOMING_PACKET_MAP.get(sourceIPPortKey);
                             long firstPacketInTime = state.getFirstPacketInTime();
                             long flowInstalledTime = state.getFlowInstalledTime();
-                            if (flowInstalledTime == -1
-                                    && (System.currentTimeMillis() - firstPacketInTime) > 4000) {
+                            if ((System.currentTimeMillis() - firstPacketInTime) > 4000) {
                                 LOG.error("onPacketReceived : Flow not installed even after 4sec."
                                         + "Dropping SNAT ({}) Packet", sourceIPPortKey);
                                 removeIncomingPacketMap(sourceIPPortKey);

diff --git a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatConstants.java b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatConstants.java
index a5e90c31ca68ee1da8758c6d8a0b64f1b9bf096a..cff89b9f557cf44d7b0d6a6c71a95987ea9dee42 100644 (file)
--- a/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatConstants.java
+++ b/vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatConstants.java
@@ -18,6 +18,7 @@ public class NatConstants {
     public static BigInteger COOKIE_NAPT_BASE = new BigInteger("8000000", 16);
     public static final String NAPT_FLOWID_PREFIX = "SNAT.";
     public static final String FLOWID_SEPARATOR = ".";
+    public static final String COLON_SEPARATOR = ":";
     public static final int DEFAULT_NAPT_IDLE_TIMEOUT = 300;
     public static int EVENT_QUEUE_LENGTH = 1000000;
     public static final String FLOWID_PREFIX = "L3.";