Stale table=44 flows remains on VM deletion 86/82986/7
authorChetan Arakere Gowdru <chetan.arakere@altencalsoftlabs.com>
Wed, 10 Jul 2019 08:30:34 +0000 (14:00 +0530)
committerFaseela K <faseela.k@ericsson.com>
Wed, 7 Aug 2019 06:48:37 +0000 (06:48 +0000)
Issue: When VM having multiple both TCP and UDP sessions are established, when
such VM is deleted, we were only taking care of deleting flows either
related to TCP or UDP session which is incorrect. This been resulting in
huge number of table=44 flows left over after deletion of VM.

Changes done.
1) Changes done to take care of cleaning of both TCP and UDP session and
it's respective flows(table 44/46) on deletion of VM.
2) Changes done to change the key on the table=44 flow to have internal
ip/port instead of external ip/port. As a result, when table=46 flow
expiry, we can still go ahead construct table=44 flow key to delete
without refering the intext-ip-port-map DS.
3) Changes done to address deletion of table 44 and table 46 flows
whenever the subnet is removed from the Router.

JIRA : NETVIRT-1611

Change-Id: I4c8502832fd64368af1d630d1fb41e89e2cb1512
Signed-off-by: Chetan Arakere Gowdru <chetan.arakere@altencalsoftlabs.com>
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NAPTEntryEvent.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptEventHandler.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatRouterInterfaceListener.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatSouthboundEventHandlers.java
natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatUtil.java

index 800e90498c3bd1b2cd62ec84fe44c4f6bf59ca8f..7417a4892842daf06bda7ddfd0f151343df8043f 100644 (file)
@@ -1257,10 +1257,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
         }
         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
-        if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
-            // Router has no interface attached
-            return;
-        }
         final long finalBgpVpnId = bgpVpnId;
         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
             List<ListenableFuture<Void>> futures = new ArrayList<>();
@@ -1269,6 +1265,10 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                     Uuid networkId = original.getNetworkId();
                     if (originalSNATEnabled != updatedSNATEnabled) {
                         if (originalSNATEnabled) {
+                            if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
+                                // Router has no interface attached
+                                return;
+                            }
                             //SNAT disabled for the router
                             Uuid networkUuid = original.getNetworkId();
                             LOG.info("update : SNAT disabled for Router {}", routerName);
@@ -1369,7 +1369,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                             LOG.debug("update : Remove the NAPT translation entries from "
                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
                             //Get the internalIP and internal Port which were associated to the removed external IP.
-                            List<Integer> externalPorts = new ArrayList<>();
                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
                                     .builder(IntextIpPortMap.class)
@@ -1391,7 +1390,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
-                                            externalPorts.add(ipPortExternal.getPortNum());
                                             List<String> removedInternalIpPorts =
                                                     protoTypesIntIpPortsMap.get(protoType);
                                             if (removedInternalIpPorts != null) {
@@ -1433,10 +1431,37 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                                         if (removedInternalPortsList != null) {
                                             removedInternalPortsList.add(removedInternalPort);
                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
+                                            naptPacketInHandler.removeIncomingPacketMap(routerId
+                                                + NatConstants.COLON_SEPARATOR + removedInternalIp
+                                                + NatConstants.COLON_SEPARATOR + removedInternalPort);
+                                            //Remove the NAPT translation entries from Outbound NAPT table
+                                            naptEventHandler.removeNatFlows(dpnId,
+                                                NwConstants.OUTBOUND_NAPT_TABLE,
+                                                routerId, removedInternalIp,
+                                                Integer.parseInt(removedInternalPort),
+                                                protocolType.getName());
+                                            naptEventHandler.removeNatFlows(dpnId,
+                                                NwConstants.INBOUND_NAPT_TABLE,
+                                                routerId, removedInternalIp,
+                                                Integer.parseInt(removedInternalPort),
+                                                protocolType.getName());
                                         } else {
                                             removedInternalPortsList = new ArrayList<>();
                                             removedInternalPortsList.add(removedInternalPort);
                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
+                                            naptPacketInHandler.removeIncomingPacketMap(routerId
+                                                + NatConstants.COLON_SEPARATOR + removedInternalIp
+                                                + NatConstants.COLON_SEPARATOR + removedInternalPort);
+                                            //Remove the NAPT translation entries from Outbound NAPT table
+                                            naptEventHandler.removeNatFlows(dpnId,
+                                                NwConstants.OUTBOUND_NAPT_TABLE,
+                                                routerId, removedInternalIp,
+                                                Integer.parseInt(removedInternalPort),
+                                                protocolType.getName());
+                                            naptEventHandler.removeNatFlows(dpnId,
+                                                NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
+                                                Integer.parseInt(removedInternalPort),
+                                                protocolType.getName());
                                         }
                                     }
                                 }
@@ -1451,30 +1476,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                             }
 
                             naptManager.removeNaptPortPool(externalIp);
-
-                            LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
-                                    + "the removed external IP {}", externalIp);
-                            for (Integer externalPort : externalPorts) {
-                                //Remove the NAPT translation entries from Inbound NAPT table
-                                naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
-                                        routerId, externalIp, externalPort);
-                            }
-
-                            Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
-                            for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
-                                String internalIp = internalIpPort.getKey();
-                                LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
-                                        + "for the removed internal IP {}", internalIp);
-                                List<String> internalPorts = internalIpPort.getValue();
-                                for (String internalPort : internalPorts) {
-                                    //Remove the NAPT translation entries from Outbound NAPT table
-                                    naptPacketInHandler.removeIncomingPacketMap(
-                                            routerId + NatConstants.COLON_SEPARATOR + internalIp
-                                                    + NatConstants.COLON_SEPARATOR + internalPort);
-                                    naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
-                                            routerId, internalIp, Integer.parseInt(internalPort));
-                                }
-                            }
                         }
                         LOG.debug(
                                 "update : End processing of the External IPs removal during the update operation");
@@ -2027,6 +2028,7 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
         }
 
         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
+            String protocol = intextIpProtocolType.getProtocol().name();
             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
                 String ipPortInternal = ipPortMap.getIpPortInternal();
                 String[] ipPortParts = ipPortInternal.split(":");
@@ -2041,7 +2043,7 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
                     + NatConstants.COLON_SEPARATOR + internalPort);
                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
-                    String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
+                    String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
                 FlowEntity outboundNaptFlowEntity =
                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
 
@@ -2049,13 +2051,9 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
 
-                IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
-                String externalIp = ipPortExternal.getIpAddress();
-                int externalPort = ipPortExternal.getPortNum();
-
-                //Build the flow for the inbound NAPT table
+                 //Build the flow for the inbound NAPT table
                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
-                    String.valueOf(routerId), externalIp, externalPort);
+                    String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
                 FlowEntity inboundNaptFlowEntity =
                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
 
@@ -2141,6 +2139,7 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                 return;
             }
             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
+                String protocol = intextIpProtocolType.getProtocol().name();
                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
                     String ipPortInternal = ipPortMap.getIpPortInternal();
                     String[] ipPortParts = ipPortInternal.split(":");
@@ -2156,7 +2155,7 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
                             + NatConstants.COLON_SEPARATOR + internalPort);
                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
-                        String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
+                        String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
                     FlowEntity outboundNaptFlowEntity =
                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
 
@@ -2166,12 +2165,11 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Rou
                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
 
                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
-                    String externalIp = ipPortExternal.getIpAddress();
-                    int externalPort = ipPortExternal.getPortNum();
+                    final String externalIp = ipPortExternal.getIpAddress();
 
                     //Build the flow for the inbound NAPT table
                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
-                        String.valueOf(routerId), externalIp, externalPort);
+                        String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
                     FlowEntity inboundNaptFlowEntity =
                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
 
index f7b6b16b93bbeafcd284a88f2ceb02ef88a4b67e..ea88ec000859f98d6784385c6ffe85dc24838ff8 100644 (file)
@@ -16,6 +16,7 @@ public class NAPTEntryEvent {
     private final String ipAddress;
     private final int portNumber;
     private final Long routerId;
+    private String flowDpn;
     private final Operation op;
     private final Protocol protocol;
     private final PacketReceived packetReceived;
@@ -36,11 +37,12 @@ public class NAPTEntryEvent {
         this.objectCreationTime = System.currentTimeMillis();
     }
 
-    NAPTEntryEvent(String ipAddress, int portNumber, Long routerId, Operation op, Protocol protocol) {
+    NAPTEntryEvent(String ipAddress, int portNumber, String flowDpn, Long routerId, Operation op, Protocol protocol) {
         this.op = op;
         this.ipAddress = ipAddress;
         this.portNumber = portNumber;
         this.routerId = routerId;
+        this.flowDpn = flowDpn;
         this.protocol = protocol;
         this.packetReceived = null;
         this.pktProcessed = false;
@@ -68,6 +70,10 @@ public class NAPTEntryEvent {
         return routerId;
     }
 
+    public String getFlowDpn() {
+        return flowDpn;
+    }
+
     public Operation getOperation() {
         return op;
     }
index 32b2dbb8f19a9e19d2b72f41c240553030ad5385..3a2e5ff79ed0ed910165f8ce71ba5505fd9c3a1c 100644 (file)
@@ -99,7 +99,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.N
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketProcessingService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.TransmitPacketInput;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
@@ -170,6 +169,7 @@ public class NaptEventHandler {
             Long routerId = naptEntryEvent.getRouterId();
             String internalIpAddress = naptEntryEvent.getIpAddress();
             int internalPort = naptEntryEvent.getPortNumber();
+            NAPTEntryEvent.Protocol protocol = naptEntryEvent.getProtocol();
             String sourceIPPortKey = routerId + NatConstants.COLON_SEPARATOR
                 + internalIpAddress + NatConstants.COLON_SEPARATOR + internalPort;
             LOG.trace("handleEvent : Time Elapsed before procesing snat ({}:{}) packet is {} ms,routerId: {},"
@@ -241,7 +241,6 @@ public class NaptEventHandler {
                     Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 
                     SessionAddress internalAddress = new SessionAddress(internalIpAddress, internalPort);
-                    NAPTEntryEvent.Protocol protocol = naptEntryEvent.getProtocol();
 
                     //Get the external IP address for the corresponding internal IP address
                     SessionAddress externalAddress =
@@ -287,7 +286,7 @@ public class NaptEventHandler {
                                             public void onSuccess(@Nullable RpcResult<AddFlowOutput> result) {
                                                 LOG.debug("handleEvent : Configured outbound rule, sending packet out"
                                                         + "from {} to {}", internalAddress, externalAddress);
-                                                prepareAndSendPacketOut(naptEntryEvent, finalRouterId);
+                                                prepareAndSendPacketOut(naptEntryEvent, finalRouterId, sourceIPPortKey);
                                             }
 
                                             @Override
@@ -307,7 +306,7 @@ public class NaptEventHandler {
                                 }
                             }, MoreExecutors.directExecutor());
                 } else {
-                    prepareAndSendPacketOut(naptEntryEvent, routerId);
+                    prepareAndSendPacketOut(naptEntryEvent, routerId, sourceIPPortKey);
                 }
                 LOG.trace("handleEvent : Time elapsed after Processsing snat ({}:{}) packet: {}ms,isPktProcessed:{} ",
                         naptEntryEvent.getIpAddress(), naptEntryEvent.getPortNumber(),
@@ -324,7 +323,7 @@ public class NaptEventHandler {
         }
     }
 
-    private void prepareAndSendPacketOut(NAPTEntryEvent naptEntryEvent, Long routerId) {
+    private void prepareAndSendPacketOut(NAPTEntryEvent naptEntryEvent, Long routerId, String sourceIPPortKey) {
         //Send Packetout - tcp or udp packets which got punted to controller.
         BigInteger metadata = naptEntryEvent.getPacketReceived().getMatch().getMetadata().getMetadata();
         byte[] inPayload = naptEntryEvent.getPacketReceived().getPayload();
@@ -333,6 +332,7 @@ public class NaptEventHandler {
             try {
                 ethPkt.deserialize(inPayload, 0, inPayload.length * Byte.SIZE);
             } catch (PacketException e) {
+                NaptPacketInHandler.removeIncomingPacketMap(sourceIPPortKey);
                 LOG.error("prepareAndSendPacketOut : Failed to decode Packet", e);
                 return;
             }
@@ -351,6 +351,7 @@ public class NaptEventHandler {
             int vlanId = 0;
             iface = interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName);
             if (iface == null) {
+                NaptPacketInHandler.removeIncomingPacketMap(sourceIPPortKey);
                 LOG.error("prepareAndSendPacketOut : Unable to read interface {} from config DataStore", interfaceName);
                 return;
             }
@@ -445,9 +446,14 @@ public class NaptEventHandler {
         int translatedPort = translatedSourceAddress.getPortNumber();
         String actualIp = actualSourceAddress.getIpAddress();
         int actualPort = actualSourceAddress.getPortNumber();
-        String switchFlowRef =
-            NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), actualIp, actualPort);
-
+        String switchFlowRef = null;
+        if (tableId == NwConstants.OUTBOUND_NAPT_TABLE) {
+            switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), actualIp, actualPort,
+                protocol.name());
+        } else {
+            switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), translatedIp,
+                translatedPort, protocol.name());
+        }
         FlowEntity snatFlowEntity = new FlowEntityBuilder()
             .setDpnId(dpnId)
             .setTableId(tableId)
@@ -625,7 +631,7 @@ public class NaptEventHandler {
         return instructionInfo;
     }
 
-    void removeNatFlows(BigInteger dpnId, short tableId ,long segmentId, String ip, int port) {
+    void removeNatFlows(BigInteger dpnId, short tableId ,long segmentId, String ip, int port, String protocol) {
         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
             LOG.error("removeNatFlows : DPN ID {} is invalid" , dpnId);
             return;
@@ -634,7 +640,7 @@ public class NaptEventHandler {
             dpnId, segmentId, ip, port);
 
         //Build the flow with the port IP and port as the match info.
-        String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(segmentId), ip, port);
+        String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(segmentId), ip, port, protocol);
         FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef);
         LOG.debug("removeNatFlows : Remove the flow in the table {} for the switch with the DPN ID {}",
                 tableId, dpnId);
@@ -647,7 +653,7 @@ public class NaptEventHandler {
     @Nullable
     @SuppressFBWarnings("PZLA_PREFER_ZERO_LENGTH_ARRAYS")
     protected byte[] buildNaptPacketOut(Ethernet etherPkt) {
-        LOG.debug("removeNatFlows : About to build Napt Packet Out");
+        LOG.debug("buildNaptPacketOut : About to build Napt Packet Out");
         if (etherPkt.getPayload() instanceof IPv4) {
             byte[] rawPkt;
             IPv4 ipPkt = (IPv4) etherPkt.getPayload();
@@ -660,11 +666,11 @@ public class NaptEventHandler {
                     return null;
                 }
             } else {
-                LOG.error("removeNatFlows : Unable to build NaptPacketOut since its neither TCP nor UDP");
+                LOG.error("buildNaptPacketOut : Unable to build NaptPacketOut since its neither TCP nor UDP");
                 return null;
             }
         }
-        LOG.error("removeNatFlows : Unable to build NaptPacketOut since its not IPv4 packet");
+        LOG.error("buildNaptPacketOut : Unable to build NaptPacketOut since its not IPv4 packet");
         return null;
     }
 
@@ -720,19 +726,17 @@ public class NaptEventHandler {
         Integer internalPortNumber = naptEntryEvent.getPortNumber();
         NAPTEntryEvent.Protocol protocol = naptEntryEvent.getProtocol();
         //Get the external IP address and the port from the model
-        IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId, internalIpv4HostAddress,
-                internalPortNumber.toString(), protocol);
-        if (ipPortExternal == null) {
-            LOG.error("handleFlowRemoved : IpPortExternal is null while queried from the model for routerId {}",
-                    routerId);
-            return;
-        }
-        String externalIpAddress = ipPortExternal.getIpAddress();
-        int externalPortNumber = ipPortExternal.getPortNum();
-
-        removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, routerId, externalIpAddress, externalPortNumber);
-
-        removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, internalIpv4HostAddress, internalPortNumber);
+        LOG.trace("handleFlowRemoved: Failed to remove snat flow internalIP {} with "
+                + "Port {} protocol {} for routerId {} in OUTBOUNDTABLE of naptSwitch {}",
+            internalIpv4HostAddress, internalPortNumber, protocol, routerId, dpnId);
+        removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, internalIpv4HostAddress,
+            internalPortNumber, protocol.name());
+
+        LOG.trace("handleFlowRemoved: Failed to remove snat flow internalIP {} with "
+                + "Port {} protocol {} for routerId {} in INBOUNDTABLE of naptSwitch {}",
+            internalIpv4HostAddress, internalPortNumber, protocol, routerId, dpnId);
+        removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, routerId, internalIpv4HostAddress,
+            internalPortNumber, protocol.name());
 
         //Remove the SourceIP:Port key from the Napt packet handler map.
         NaptPacketInHandler.removeIncomingPacketMap(sourceIPPortKey);
index f5d614cb9af990e8830bec4c9bff1366eb16c564..e2c789f76456ba9a3e40ff701069830f37752c7a 100644 (file)
@@ -121,8 +121,9 @@ public class NaptFlowRemovedEventHandler implements SalFlowListener {
                 LOG.error("onFlowRemoved : Null exception while retrieving routerId");
                 return;
             }
-            NAPTEntryEvent naptEntryEvent = new NAPTEntryEvent(internalIpv4HostAddress, internalPortNumber, routerId,
-                    NAPTEntryEvent.Operation.DELETE, protocol);
+            String flowDpn = NatUtil.getDpnFromNodeRef(flowRemoved.getNode());
+            NAPTEntryEvent naptEntryEvent = new NAPTEntryEvent(internalIpv4HostAddress, internalPortNumber, flowDpn,
+                routerId, NAPTEntryEvent.Operation.DELETE, protocol);
             naptEventdispatcher.addFlowRemovedNaptEvent(naptEntryEvent);
         } else {
             LOG.debug("onFlowRemoved : Received flow removed notification due to flowdelete from switch for flowref");
@@ -139,13 +140,10 @@ public class NaptFlowRemovedEventHandler implements SalFlowListener {
     @Override
     public void onNodeErrorNotification(NodeErrorNotification arg0) {
         // TODO Auto-generated method stub
-
     }
 
     @Override
     public void onNodeExperimenterErrorNotification(NodeExperimenterErrorNotification arg0) {
         // TODO Auto-generated method stub
-
     }
-
 }
index ca6336a0cb44262e1f0c373e934b26bad4022bea..a81d15f5f7c7862b5a75885a74f5a15f0ddeaeca 100644 (file)
@@ -308,8 +308,9 @@ public class NaptSwitchHA {
                 LOG.debug("removeSnatFlowsInOldNaptSwitch : No {} session associated to router {},"
                         + "no flows need to be removed in oldNaptSwitch {}",
                     intextIpProtocolType.getProtocol(), routerId, naptSwitch);
-                break;
+                continue;
             }
+            String protocol = intextIpProtocolType.getProtocol().name();
             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
             for (IpPortMap ipPortMap : ipPortMaps) {
                 String ipPortInternal = ipPortMap.getIpPortInternal();
@@ -324,7 +325,7 @@ public class NaptSwitchHA {
                 //Build and remove flow in outbound NAPT table
                 String switchFlowRef =
                     NatUtil.getNaptFlowRef(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, String.valueOf(routerId),
-                        internalIp, Integer.parseInt(internalPort));
+                        internalIp, Integer.parseInt(internalPort), protocol);
                 FlowEntity outboundNaptFlowEntity =
                     NatUtil.buildFlowEntity(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
                         cookieSnatFlow, switchFlowRef);
@@ -333,20 +334,10 @@ public class NaptSwitchHA {
                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, naptSwitch, routerId);
                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
 
-                IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
-                if (ipPortExternal == null) {
-                    LOG.debug(
-                        "removeSnatFlowsInOldNaptSwitch : External Ipport mapping not found for internalIp {} "
-                            + "with port {} for router {}", internalIp, internalPort, routerId);
-                    continue;
-                }
-                String externalIp = ipPortExternal.getIpAddress();
-                int externalPort = ipPortExternal.getPortNum();
-
                 //Build and remove flow in  inbound NAPT table
                 switchFlowRef =
                     NatUtil.getNaptFlowRef(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, String.valueOf(routerId),
-                        externalIp, externalPort);
+                        internalIp, Integer.parseInt(internalPort), protocol);
                 FlowEntity inboundNaptFlowEntity =
                     NatUtil.buildFlowEntity(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
                         cookieSnatFlow, switchFlowRef);
index beece9dd1210430d40390e1e03bc90cb79c38b58..19b043d2e165b7e719e621b801b7d7b9f721c124 100644 (file)
@@ -21,11 +21,13 @@ import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.mdsalutil.MDSALUtil;
+import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterfaceBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterfaceKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterInterfacesMap;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.router.interfaces.map.RouterInterfaces;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.router.interfaces.map.router.interfaces.Interfaces;
@@ -41,13 +43,21 @@ public class NatRouterInterfaceListener
     private final DataBroker dataBroker;
     private final ManagedNewTransactionRunner txRunner;
     private final OdlInterfaceRpcService interfaceManager;
+    private final IMdsalApiManager mdsalManager;
+    private final NaptManager naptManager;
+    private final NeutronvpnService neutronVpnService;
 
     @Inject
-    public NatRouterInterfaceListener(final DataBroker dataBroker, final OdlInterfaceRpcService interfaceManager) {
+    public NatRouterInterfaceListener(final DataBroker dataBroker, final OdlInterfaceRpcService interfaceManager,
+        final IMdsalApiManager mdsalManager,final NaptManager naptManager,
+        final NeutronvpnService neutronvpnService) {
         super(Interfaces.class, NatRouterInterfaceListener.class);
         this.dataBroker = dataBroker;
         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
         this.interfaceManager = interfaceManager;
+        this.mdsalManager = mdsalManager;
+        this.naptManager = naptManager;
+        this.neutronVpnService = neutronvpnService;
     }
 
     @Override
@@ -130,6 +140,10 @@ public class NatRouterInterfaceListener
         final ReentrantLock lock = NatUtil.lockForNat(dpId);
         lock.lock();
         try {
+            if (NatUtil.isSnatEnabledForRouterId(dataBroker, routerId)) {
+                NatUtil.removeSnatEntriesForPort(dataBroker, naptManager, mdsalManager, neutronVpnService,
+                    interfaceName, routerId);
+            }
             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
                 //Delete the NeutronRouterDpnMap from the ODL:L3VPN operational model
                 NatUtil.removeFromNeutronRouterDpnsMap(routerId, interfaceName, dpId, operTx);
index 6bcc99185bc48f0df77d013e29139b1e85d7081a..881ff6d558b99c597dc59a752fdb164b06154907 100644 (file)
@@ -22,7 +22,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.concurrent.locks.ReentrantLock;
 import javax.inject.Inject;
 import javax.inject.Singleton;
@@ -35,30 +34,17 @@ import org.opendaylight.genius.infra.Datastore.Operational;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
-import org.opendaylight.genius.mdsalutil.FlowEntity;
-import org.opendaylight.genius.mdsalutil.NwConstants;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
-import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterface;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.RouterPorts;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.Ports;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.neutron.vip.states.VipState;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.IpPort;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.ip.port.IntIpProtoType;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortInputBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortOutput;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
-import org.opendaylight.yangtools.yang.common.RpcResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -324,51 +310,6 @@ public class NatSouthboundEventHandlers {
         return port.get().nonnullInternalToExternalPortMap();
     }
 
-    @Nullable
-    private BigInteger getNaptSwitchforRouter(DataBroker broker, String routerName) {
-        InstanceIdentifier<RouterToNaptSwitch> rtrNaptSw = InstanceIdentifier.builder(NaptSwitches.class)
-            .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
-        Optional<RouterToNaptSwitch> routerToNaptSwitchData =
-                SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(broker,
-                        LogicalDatastoreType.CONFIGURATION, rtrNaptSw);
-        if (routerToNaptSwitchData.isPresent()) {
-            RouterToNaptSwitch routerToNaptSwitchInstance = routerToNaptSwitchData.get();
-            return routerToNaptSwitchInstance.getPrimarySwitchId();
-        }
-        return null;
-    }
-
-    private void removeNatFlow(BigInteger dpnId, short tableId, Long routerId, String ipAddress, int ipPort) {
-
-        String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), ipAddress, ipPort);
-        FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef);
-
-        mdsalManager.removeFlow(snatFlowEntity);
-        LOG.debug("removeNatFlow : Removed the flow in table {} for the switch with the DPN ID {} for "
-            + "router {} ip {} port {}", tableId, dpnId, routerId, ipAddress, ipPort);
-    }
-
-    @Nullable
-    private List<String> getFixedIpsForPort(String interfname) {
-        LOG.debug("getFixedIpsForPort : getFixedIpsForPort method is called for interface {}", interfname);
-        try {
-            Future<RpcResult<GetFixedIPsForNeutronPortOutput>> result =
-                neutronVpnService.getFixedIPsForNeutronPort(new GetFixedIPsForNeutronPortInputBuilder()
-                    .setPortId(new Uuid(interfname)).build());
-
-            RpcResult<GetFixedIPsForNeutronPortOutput> rpcResult = result.get();
-            if (!rpcResult.isSuccessful()) {
-                LOG.error("getFixedIpsForPort : RPC Call to GetFixedIPsForNeutronPortOutput returned with Errors {}",
-                    rpcResult.getErrors());
-            } else {
-                return rpcResult.getResult().getFixedIPs();
-            }
-        } catch (InterruptedException | ExecutionException | NullPointerException ex) {
-            LOG.error("getFixedIpsForPort : Exception while receiving fixedIps for port {}", interfname, ex);
-        }
-        return null;
-    }
-
     private void processInterfaceRemoved(String portName, BigInteger dpnId, String routerId,
             List<ListenableFuture<Void>> futures) {
         LOG.trace("processInterfaceRemoved : Processing Interface Removed Event for interface {} on DPN ID {}",
@@ -393,89 +334,6 @@ public class NatSouthboundEventHandlers {
         }
     }
 
-    // TODO Clean up the exception handling
-    @SuppressWarnings("checkstyle:IllegalCatch")
-    private void removeSnatEntriesForPort(String interfaceName, String routerName) {
-        Long routerId = NatUtil.getVpnId(dataBroker, routerName);
-        if (routerId == NatConstants.INVALID_ID) {
-            LOG.error("removeSnatEntriesForPort : routerId not found for routername {}", routerName);
-            return;
-        }
-        BigInteger naptSwitch = getNaptSwitchforRouter(dataBroker, routerName);
-        if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
-            LOG.error("removeSnatEntriesForPort : NaptSwitch is not elected for router {} with Id {}",
-                    routerName, routerId);
-            return;
-        }
-        //getInternalIp for port
-        List<String> fixedIps = getFixedIpsForPort(interfaceName);
-        if (fixedIps == null) {
-            LOG.warn("removeSnatEntriesForPort : Internal Ips not found for InterfaceName {} in router {} with id {}",
-                interfaceName, routerName, routerId);
-            return;
-        }
-
-        for (String internalIp : fixedIps) {
-            LOG.debug("removeSnatEntriesForPort : Internal Ip retrieved for interface {} is {} in router with Id {}",
-                interfaceName, internalIp, routerId);
-            IpPort ipPort = NatUtil.getInternalIpPortInfo(dataBroker, routerId, internalIp);
-            if (ipPort == null) {
-                LOG.debug("removeSnatEntriesForPort : no snatint-ip-port-map found for ip:{}", internalIp);
-                continue;
-            }
-
-            for (IntIpProtoType protoType : ipPort.nonnullIntIpProtoType()) {
-                ProtocolTypes protocol = protoType.getProtocol();
-                @Nullable List<Integer> ports = protoType.getPorts();
-                for (Integer portnum : (ports != null ? ports : Collections.<Integer>emptyList())) {
-                    //build and remove the flow in outbound table
-                    try {
-                        removeNatFlow(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, routerId, internalIp, portnum);
-                    } catch (Exception ex) {
-                        LOG.error("removeSnatEntriesForPort : Failed to remove snat flow for internalIP {} with "
-                                + "Port {} protocol {} for routerId {} in OUTBOUNDTABLE of NaptSwitch {}",
-                            internalIp, portnum, protocol, routerId, naptSwitch, ex);
-                    }
-                    //Get the external IP address and the port from the model
-                    NAPTEntryEvent.Protocol proto = protocol.toString().equals(ProtocolTypes.TCP.toString())
-                        ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP;
-                    IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId,
-                        internalIp, String.valueOf(portnum), proto);
-                    if (ipPortExternal == null) {
-                        LOG.error("removeSnatEntriesForPort : Mapping for internalIp {} with port {} is not found in "
-                            + "router with Id {}", internalIp, portnum, routerId);
-                        return;
-                    }
-                    String externalIpAddress = ipPortExternal.getIpAddress();
-                    Integer portNumber = ipPortExternal.getPortNum();
-
-                    //build and remove the flow in inboundtable
-                    try {
-                        removeNatFlow(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, routerId,
-                            externalIpAddress, portNumber);
-                    } catch (Exception ex) {
-                        LOG.error("removeSnatEntriesForPort : Failed to remove snat flow internalIP {} with "
-                                + "Port {} protocol {} for routerId {} in INBOUNDTABLE of naptSwitch {}",
-                            externalIpAddress, portNumber, protocol, routerId, naptSwitch, ex);
-                    }
-
-                    String internalIpPort = internalIp + ":" + portnum;
-                    // delete the entry from IntExtIpPortMap DS
-                    try {
-                        naptManager.removeFromIpPortMapDS(routerId, internalIpPort, proto);
-                        naptManager.removePortFromPool(internalIpPort, externalIpAddress);
-                    } catch (Exception ex) {
-                        LOG.error("removeSnatEntriesForPort : releaseIpExtPortMapping failed, Removal of "
-                            + "ipportmap {} for router {} failed", internalIpPort, routerId, ex);
-                    }
-                }
-            }
-            // delete the entry from SnatIntIpPortMap DS
-            LOG.debug("removeSnatEntriesForPort : Removing InternalIp:{} on router {}", internalIp, routerId);
-            naptManager.removeFromSnatIpPortDS(routerId, internalIp);
-        }
-    }
-
     private class NatFlowAddWorker implements Callable<List<ListenableFuture<Void>>> {
         private final String interfaceName;
         private final String routerName;
@@ -532,7 +390,10 @@ public class NatSouthboundEventHandlers {
             } else if (state.equals(IntfTransitionState.STATE_DOWN)) {
                 LOG.debug("call : Port DOWN event received for interface {} ", interfaceName);
                 try {
-                    removeSnatEntriesForPort(interfaceName, routerName);
+                    if (NatUtil.isSnatEnabledForRouterId(dataBroker, routerName)) {
+                        NatUtil.removeSnatEntriesForPort(dataBroker, naptManager, mdsalManager, neutronVpnService,
+                            interfaceName, routerName);
+                    }
                 } catch (Exception ex) {
                     LOG.error("call : Exception caught in Interface {} OperationalStateDown", interfaceName, ex);
                 }
@@ -560,7 +421,10 @@ public class NatSouthboundEventHandlers {
             try {
                 LOG.trace("call : Port removed event received for interface {} ", interfaceName);
                 processInterfaceRemoved(interfaceName, intfDpnId, routerName, futures);
-                removeSnatEntriesForPort(interfaceName, routerName);
+                if (NatUtil.isSnatEnabledForRouterId(dataBroker, routerName)) {
+                    NatUtil.removeSnatEntriesForPort(dataBroker, naptManager, mdsalManager, neutronVpnService,
+                        interfaceName, routerName);
+                }
             } catch (Exception e) {
                 LOG.error("call : Exception caught in Interface {} OperationalStateRemove", interfaceName, e);
             }
index c53b27fb6396a3ee6e2cf95d08f94dd78351c30f..7b72c484ee756a9396cc4b16209c678e6cb507cb 100644 (file)
@@ -16,6 +16,7 @@ import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.math.BigInteger;
 import java.net.InetAddress;
@@ -40,6 +41,7 @@ import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
+import org.opendaylight.controller.sal.common.util.Arguments;
 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
 import org.opendaylight.genius.infra.Datastore.Configuration;
 import org.opendaylight.genius.infra.Datastore.Operational;
@@ -131,6 +133,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group
 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
@@ -231,8 +234,11 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev16011
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.IpPortKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.ip.port.IntIpProtoType;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.ip.port.IntIpProtoTypeKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortInputBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortOutput;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkMaps;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronVpnPortipPortData;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.VpnMaps;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.networkmaps.NetworkMap;
@@ -260,6 +266,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.re
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.OpenvswitchOtherConfigs;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.PathArgument;
 import org.opendaylight.yangtools.yang.common.RpcResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -394,19 +401,23 @@ public final class NatUtil {
        getFlowRef() returns a string identfier for the SNAT flows using the router ID as the reference.
     */
     public static String getFlowRef(BigInteger dpnId, short tableId, long routerID, String ip) {
-        return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
-                .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + ip;
+        return new StringBuilder().append(NatConstants.NAPT_FLOWID_PREFIX).append(dpnId)
+            .append(NatConstants.FLOWID_SEPARATOR).append(tableId).append(NatConstants.FLOWID_SEPARATOR)
+            .append(routerID).append(NatConstants.FLOWID_SEPARATOR).append(ip).toString();
     }
 
     public static String getFlowRef(BigInteger dpnId, short tableId, InetAddress destPrefix, long vpnId) {
-        return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
-                .FLOWID_SEPARATOR + destPrefix.getHostAddress() + NatConstants.FLOWID_SEPARATOR + vpnId;
+        return new StringBuilder().append(NatConstants.NAPT_FLOWID_PREFIX).append(dpnId)
+            .append(NatConstants.FLOWID_SEPARATOR).append(tableId).append(NatConstants.FLOWID_SEPARATOR)
+            .append(destPrefix.getHostAddress()).append(NatConstants.FLOWID_SEPARATOR).append(vpnId).toString();
     }
 
-    public static String getNaptFlowRef(BigInteger dpnId, short tableId, String routerID, String ip, int port) {
-        return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
-                .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + ip + NatConstants.FLOWID_SEPARATOR
-                + port;
+    public static String getNaptFlowRef(BigInteger dpnId, short tableId, String routerID, String ip,
+        int port, String protocol) {
+        return new StringBuilder().append(NatConstants.NAPT_FLOWID_PREFIX).append(dpnId)
+            .append(NatConstants.FLOWID_SEPARATOR).append(tableId).append(NatConstants.FLOWID_SEPARATOR)
+            .append(routerID).append(NatConstants.FLOWID_SEPARATOR).append(ip).append(NatConstants.FLOWID_SEPARATOR)
+            .append(port).append(NatConstants.FLOWID_SEPARATOR).append(protocol).toString();
     }
 
     @Nullable
@@ -2567,4 +2578,127 @@ public final class NatUtil {
         // FIXME: wrap this in an Identifier
         return JvmGlobalLocks.getLockForString(NatConstants.NAT_DJC_PREFIX + dataPath);
     }
+
+    public static void removeSnatEntriesForPort(DataBroker dataBroker, NaptManager naptManager,
+        IMdsalApiManager mdsalManager, NeutronvpnService neutronVpnService,
+        String interfaceName, String routerName) {
+        Long routerId = NatUtil.getVpnId(dataBroker, routerName);
+        if (routerId == NatConstants.INVALID_ID) {
+            LOG.error("removeSnatEntriesForPort: routerId not found for routername {}", routerName);
+            return;
+        }
+        BigInteger naptSwitch = getPrimaryNaptfromRouterName(dataBroker, routerName);
+        if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
+            LOG.error("removeSnatEntriesForPort: NaptSwitch is not elected for router {}"
+                + "with Id {}", routerName, routerId);
+            return;
+        }
+        //getInternalIp for port
+        List<String> fixedIps = getFixedIpsForPort(neutronVpnService, interfaceName);
+        if (fixedIps == null) {
+            LOG.error("removeSnatEntriesForPort: Internal Ips not found for InterfaceName {} in router {} with id {}",
+                interfaceName, routerName, routerId);
+            return;
+        }
+        List<ProtocolTypes> protocolTypesList = getPortocolList();
+        for (String internalIp : fixedIps) {
+            LOG.debug("removeSnatEntriesForPort: Internal Ip retrieved for interface {} is {} in router with Id {}",
+                interfaceName, internalIp, routerId);
+            for (ProtocolTypes protocol : protocolTypesList) {
+                List<Integer> portList = NatUtil.getInternalIpPortListInfo(dataBroker, routerId, internalIp, protocol);
+                if (portList != null) {
+                    for (Integer portnum : portList) {
+                        //build and remove the flow in outbound table
+                        removeNatFlow(mdsalManager, naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
+                            routerId, internalIp, portnum, protocol.getName());
+
+                        //build and remove the flow in inboundtable
+
+                        removeNatFlow(mdsalManager, naptSwitch, NwConstants.INBOUND_NAPT_TABLE, routerId,
+                            internalIp, portnum, protocol.getName());
+
+                        //Get the external IP address and the port from the model
+
+                        NAPTEntryEvent.Protocol proto = protocol.toString().equals(ProtocolTypes.TCP.toString())
+                            ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP;
+                        IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId,
+                            internalIp, String.valueOf(portnum), proto);
+                        if (ipPortExternal == null) {
+                            LOG.error("removeSnatEntriesForPort: Mapping for internalIp {} "
+                                + "with port {} is not found in "
+                                + "router with Id {}", internalIp, portnum, routerId);
+                            return;
+                        }
+                        String externalIpAddress = ipPortExternal.getIpAddress();
+                        String internalIpPort = internalIp + ":" + portnum;
+                        // delete the entry from IntExtIpPortMap DS
+
+                        naptManager.removeFromIpPortMapDS(routerId, internalIpPort, proto);
+                        naptManager.removePortFromPool(internalIpPort, externalIpAddress);
+
+                    }
+                } else {
+                    LOG.debug("removeSnatEntriesForPort: No {} session for interface {} with internalIP {} "
+                            + "in router with id {}",
+                        protocol, interfaceName, internalIp, routerId);
+                }
+            }
+            // delete the entry from SnatIntIpPortMap DS
+            LOG.debug("removeSnatEntriesForPort: Removing InternalIp :{} of router {} from snatint-ip-port-map",
+                internalIp, routerId);
+            naptManager.removeFromSnatIpPortDS(routerId, internalIp);
+        }
+    }
+
+    private static List<String> getFixedIpsForPort(NeutronvpnService neutronVpnService, String interfname) {
+        LOG.debug("getFixedIpsForPort: getFixedIpsForPort method is called for interface {}", interfname);
+        try {
+            Future<RpcResult<GetFixedIPsForNeutronPortOutput>> result =
+                neutronVpnService.getFixedIPsForNeutronPort(new GetFixedIPsForNeutronPortInputBuilder()
+                    .setPortId(new Uuid(interfname)).build());
+
+            RpcResult<GetFixedIPsForNeutronPortOutput> rpcResult = result.get();
+            if (!rpcResult.isSuccessful()) {
+                LOG.error("getFixedIpsForPort: RPC Call to GetFixedIPsForNeutronPortOutput returned with Errors {}",
+                    rpcResult.getErrors());
+            } else {
+                return rpcResult.getResult().getFixedIPs();
+            }
+        } catch (InterruptedException | ExecutionException | NullPointerException ex) {
+            LOG.error("getFixedIpsForPort: Exception while receiving fixedIps for port {}", interfname, ex);
+        }
+        return null;
+    }
+
+    private static List<ProtocolTypes> getPortocolList() {
+        List<ProtocolTypes> protocollist = new ArrayList<>();
+        protocollist.add(ProtocolTypes.TCP);
+        protocollist.add(ProtocolTypes.UDP);
+        return protocollist;
+    }
+
+    private static void removeNatFlow(IMdsalApiManager mdsalManager, BigInteger dpnId, short tableId, Long routerId,
+        String ipAddress, int ipPort, String protocol) {
+
+        String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), ipAddress, ipPort,
+            protocol);
+        FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef);
+
+        mdsalManager.removeFlow(snatFlowEntity);
+        LOG.debug("removeNatFlow: Removed the flow in table {} for the switch with the DPN ID {} for "
+            + "router {} ip {} port {}", tableId, dpnId, routerId, ipAddress, ipPort);
+    }
+
+    public static String getDpnFromNodeRef(NodeRef node) {
+        PathArgument pathArgument = Iterables.get(node.getValue().getPathArguments(), 1);
+        InstanceIdentifier.IdentifiableItem<?, ?> item = Arguments.checkInstanceOf(pathArgument,
+            InstanceIdentifier.IdentifiableItem.class);
+        NodeKey key = Arguments.checkInstanceOf(item.getKey(), NodeKey.class);
+        String dpnKey = key.getId().getValue();
+        String dpnID = null;
+        if (dpnKey.contains(NatConstants.COLON_SEPARATOR)) {
+            dpnID = new BigInteger(dpnKey.split(NatConstants.COLON_SEPARATOR)[1]).toString();
+        }
+        return dpnID;
+    }
 }