From 3b23f00dfc42b7ca8c5bc2be202ecd0b6bfc9619 Mon Sep 17 00:00:00 2001 From: Chetan Arakere Gowdru Date: Wed, 10 Jul 2019 14:00:34 +0530 Subject: [PATCH] Stale table=44 flows remains on VM deletion Issue: When VM having multiple both TCP and UDP sessions are established, when such VM is deleted, we were only taking care of deleting flows either related to TCP or UDP session which is incorrect. This been resulting in huge number of table=44 flows left over after deletion of VM. Changes done. 1) Changes done to take care of cleaning of both TCP and UDP session and it's respective flows(table 44/46) on deletion of VM. 2) Changes done to change the key on the table=44 flow to have internal ip/port instead of external ip/port. As a result, when table=46 flow expiry, we can still go ahead construct table=44 flow key to delete without refering the intext-ip-port-map DS. 3) Changes done to address deletion of table 44 and table 46 flows whenever the subnet is removed from the Router. JIRA : NETVIRT-1611 Change-Id: I4c8502832fd64368af1d630d1fb41e89e2cb1512 Signed-off-by: Chetan Arakere Gowdru --- .../internal/ExternalRoutersListener.java | 80 +++++---- .../natservice/internal/NAPTEntryEvent.java | 8 +- .../natservice/internal/NaptEventHandler.java | 56 ++++--- .../internal/NaptFlowRemovedEventHandler.java | 8 +- .../natservice/internal/NaptSwitchHA.java | 17 +- .../internal/NatRouterInterfaceListener.java | 16 +- .../internal/NatSouthboundEventHandlers.java | 152 +----------------- .../netvirt/natservice/internal/NatUtil.java | 150 ++++++++++++++++- 8 files changed, 248 insertions(+), 239 deletions(-) diff --git a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java index 800e90498c..7417a48928 100644 --- a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java +++ b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java @@ -1257,10 +1257,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase { List> futures = new ArrayList<>(); @@ -1269,6 +1265,10 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase externalPorts = new ArrayList<>(); Map> protoTypesIntIpPortsMap = new HashMap<>(); InstanceIdentifier ipPortMappingId = InstanceIdentifier .builder(IntextIpPortMap.class) @@ -1391,7 +1390,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase removedInternalIpPorts = protoTypesIntIpPortsMap.get(protoType); if (removedInternalIpPorts != null) { @@ -1433,10 +1431,37 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase(); removedInternalPortsList.add(removedInternalPort); internalIpPortMap.put(removedInternalIp, removedInternalPortsList); + naptPacketInHandler.removeIncomingPacketMap(routerId + + NatConstants.COLON_SEPARATOR + removedInternalIp + + NatConstants.COLON_SEPARATOR + removedInternalPort); + //Remove the NAPT translation entries from Outbound NAPT table + naptEventHandler.removeNatFlows(dpnId, + NwConstants.OUTBOUND_NAPT_TABLE, + routerId, removedInternalIp, + Integer.parseInt(removedInternalPort), + protocolType.getName()); + naptEventHandler.removeNatFlows(dpnId, + NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp, + Integer.parseInt(removedInternalPort), + protocolType.getName()); } } } @@ -1451,30 +1476,6 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase>> internalIpPorts = internalIpPortMap.entrySet(); - for (Map.Entry> internalIpPort : internalIpPorts) { - String internalIp = internalIpPort.getKey(); - LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables " - + "for the removed internal IP {}", internalIp); - List internalPorts = internalIpPort.getValue(); - for (String internalPort : internalPorts) { - //Remove the NAPT translation entries from Outbound NAPT table - naptPacketInHandler.removeIncomingPacketMap( - routerId + NatConstants.COLON_SEPARATOR + internalIp - + NatConstants.COLON_SEPARATOR + internalPort); - naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, - routerId, internalIp, Integer.parseInt(internalPort)); - } - } } LOG.debug( "update : End processing of the External IPs removal during the update operation"); @@ -2027,6 +2028,7 @@ public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase result) { LOG.debug("handleEvent : Configured outbound rule, sending packet out" + "from {} to {}", internalAddress, externalAddress); - prepareAndSendPacketOut(naptEntryEvent, finalRouterId); + prepareAndSendPacketOut(naptEntryEvent, finalRouterId, sourceIPPortKey); } @Override @@ -307,7 +306,7 @@ public class NaptEventHandler { } }, MoreExecutors.directExecutor()); } else { - prepareAndSendPacketOut(naptEntryEvent, routerId); + prepareAndSendPacketOut(naptEntryEvent, routerId, sourceIPPortKey); } LOG.trace("handleEvent : Time elapsed after Processsing snat ({}:{}) packet: {}ms,isPktProcessed:{} ", naptEntryEvent.getIpAddress(), naptEntryEvent.getPortNumber(), @@ -324,7 +323,7 @@ public class NaptEventHandler { } } - private void prepareAndSendPacketOut(NAPTEntryEvent naptEntryEvent, Long routerId) { + private void prepareAndSendPacketOut(NAPTEntryEvent naptEntryEvent, Long routerId, String sourceIPPortKey) { //Send Packetout - tcp or udp packets which got punted to controller. BigInteger metadata = naptEntryEvent.getPacketReceived().getMatch().getMetadata().getMetadata(); byte[] inPayload = naptEntryEvent.getPacketReceived().getPayload(); @@ -333,6 +332,7 @@ public class NaptEventHandler { try { ethPkt.deserialize(inPayload, 0, inPayload.length * Byte.SIZE); } catch (PacketException e) { + NaptPacketInHandler.removeIncomingPacketMap(sourceIPPortKey); LOG.error("prepareAndSendPacketOut : Failed to decode Packet", e); return; } @@ -351,6 +351,7 @@ public class NaptEventHandler { int vlanId = 0; iface = interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName); if (iface == null) { + NaptPacketInHandler.removeIncomingPacketMap(sourceIPPortKey); LOG.error("prepareAndSendPacketOut : Unable to read interface {} from config DataStore", interfaceName); return; } @@ -445,9 +446,14 @@ public class NaptEventHandler { int translatedPort = translatedSourceAddress.getPortNumber(); String actualIp = actualSourceAddress.getIpAddress(); int actualPort = actualSourceAddress.getPortNumber(); - String switchFlowRef = - NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), actualIp, actualPort); - + String switchFlowRef = null; + if (tableId == NwConstants.OUTBOUND_NAPT_TABLE) { + switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), actualIp, actualPort, + protocol.name()); + } else { + switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), translatedIp, + translatedPort, protocol.name()); + } FlowEntity snatFlowEntity = new FlowEntityBuilder() .setDpnId(dpnId) .setTableId(tableId) @@ -625,7 +631,7 @@ public class NaptEventHandler { return instructionInfo; } - void removeNatFlows(BigInteger dpnId, short tableId ,long segmentId, String ip, int port) { + void removeNatFlows(BigInteger dpnId, short tableId ,long segmentId, String ip, int port, String protocol) { if (dpnId == null || dpnId.equals(BigInteger.ZERO)) { LOG.error("removeNatFlows : DPN ID {} is invalid" , dpnId); return; @@ -634,7 +640,7 @@ public class NaptEventHandler { dpnId, segmentId, ip, port); //Build the flow with the port IP and port as the match info. - String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(segmentId), ip, port); + String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(segmentId), ip, port, protocol); FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef); LOG.debug("removeNatFlows : Remove the flow in the table {} for the switch with the DPN ID {}", tableId, dpnId); @@ -647,7 +653,7 @@ public class NaptEventHandler { @Nullable @SuppressFBWarnings("PZLA_PREFER_ZERO_LENGTH_ARRAYS") protected byte[] buildNaptPacketOut(Ethernet etherPkt) { - LOG.debug("removeNatFlows : About to build Napt Packet Out"); + LOG.debug("buildNaptPacketOut : About to build Napt Packet Out"); if (etherPkt.getPayload() instanceof IPv4) { byte[] rawPkt; IPv4 ipPkt = (IPv4) etherPkt.getPayload(); @@ -660,11 +666,11 @@ public class NaptEventHandler { return null; } } else { - LOG.error("removeNatFlows : Unable to build NaptPacketOut since its neither TCP nor UDP"); + LOG.error("buildNaptPacketOut : Unable to build NaptPacketOut since its neither TCP nor UDP"); return null; } } - LOG.error("removeNatFlows : Unable to build NaptPacketOut since its not IPv4 packet"); + LOG.error("buildNaptPacketOut : Unable to build NaptPacketOut since its not IPv4 packet"); return null; } @@ -720,19 +726,17 @@ public class NaptEventHandler { Integer internalPortNumber = naptEntryEvent.getPortNumber(); NAPTEntryEvent.Protocol protocol = naptEntryEvent.getProtocol(); //Get the external IP address and the port from the model - IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId, internalIpv4HostAddress, - internalPortNumber.toString(), protocol); - if (ipPortExternal == null) { - LOG.error("handleFlowRemoved : IpPortExternal is null while queried from the model for routerId {}", - routerId); - return; - } - String externalIpAddress = ipPortExternal.getIpAddress(); - int externalPortNumber = ipPortExternal.getPortNum(); - - removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, routerId, externalIpAddress, externalPortNumber); - - removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, internalIpv4HostAddress, internalPortNumber); + LOG.trace("handleFlowRemoved: Failed to remove snat flow internalIP {} with " + + "Port {} protocol {} for routerId {} in OUTBOUNDTABLE of naptSwitch {}", + internalIpv4HostAddress, internalPortNumber, protocol, routerId, dpnId); + removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, internalIpv4HostAddress, + internalPortNumber, protocol.name()); + + LOG.trace("handleFlowRemoved: Failed to remove snat flow internalIP {} with " + + "Port {} protocol {} for routerId {} in INBOUNDTABLE of naptSwitch {}", + internalIpv4HostAddress, internalPortNumber, protocol, routerId, dpnId); + removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, routerId, internalIpv4HostAddress, + internalPortNumber, protocol.name()); //Remove the SourceIP:Port key from the Napt packet handler map. NaptPacketInHandler.removeIncomingPacketMap(sourceIPPortKey); diff --git a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java index f5d614cb9a..e2c789f764 100644 --- a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java +++ b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptFlowRemovedEventHandler.java @@ -121,8 +121,9 @@ public class NaptFlowRemovedEventHandler implements SalFlowListener { LOG.error("onFlowRemoved : Null exception while retrieving routerId"); return; } - NAPTEntryEvent naptEntryEvent = new NAPTEntryEvent(internalIpv4HostAddress, internalPortNumber, routerId, - NAPTEntryEvent.Operation.DELETE, protocol); + String flowDpn = NatUtil.getDpnFromNodeRef(flowRemoved.getNode()); + NAPTEntryEvent naptEntryEvent = new NAPTEntryEvent(internalIpv4HostAddress, internalPortNumber, flowDpn, + routerId, NAPTEntryEvent.Operation.DELETE, protocol); naptEventdispatcher.addFlowRemovedNaptEvent(naptEntryEvent); } else { LOG.debug("onFlowRemoved : Received flow removed notification due to flowdelete from switch for flowref"); @@ -139,13 +140,10 @@ public class NaptFlowRemovedEventHandler implements SalFlowListener { @Override public void onNodeErrorNotification(NodeErrorNotification arg0) { // TODO Auto-generated method stub - } @Override public void onNodeExperimenterErrorNotification(NodeExperimenterErrorNotification arg0) { // TODO Auto-generated method stub - } - } diff --git a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java index ca6336a0cb..a81d15f5f7 100644 --- a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java +++ b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java @@ -308,8 +308,9 @@ public class NaptSwitchHA { LOG.debug("removeSnatFlowsInOldNaptSwitch : No {} session associated to router {}," + "no flows need to be removed in oldNaptSwitch {}", intextIpProtocolType.getProtocol(), routerId, naptSwitch); - break; + continue; } + String protocol = intextIpProtocolType.getProtocol().name(); List ipPortMaps = intextIpProtocolType.getIpPortMap(); for (IpPortMap ipPortMap : ipPortMaps) { String ipPortInternal = ipPortMap.getIpPortInternal(); @@ -324,7 +325,7 @@ public class NaptSwitchHA { //Build and remove flow in outbound NAPT table String switchFlowRef = NatUtil.getNaptFlowRef(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, String.valueOf(routerId), - internalIp, Integer.parseInt(internalPort)); + internalIp, Integer.parseInt(internalPort), protocol); FlowEntity outboundNaptFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef); @@ -333,20 +334,10 @@ public class NaptSwitchHA { + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, naptSwitch, routerId); mdsalManager.removeFlow(confTx, outboundNaptFlowEntity); - IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal(); - if (ipPortExternal == null) { - LOG.debug( - "removeSnatFlowsInOldNaptSwitch : External Ipport mapping not found for internalIp {} " - + "with port {} for router {}", internalIp, internalPort, routerId); - continue; - } - String externalIp = ipPortExternal.getIpAddress(); - int externalPort = ipPortExternal.getPortNum(); - //Build and remove flow in inbound NAPT table switchFlowRef = NatUtil.getNaptFlowRef(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, String.valueOf(routerId), - externalIp, externalPort); + internalIp, Integer.parseInt(internalPort), protocol); FlowEntity inboundNaptFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef); diff --git a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatRouterInterfaceListener.java b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatRouterInterfaceListener.java index beece9dd12..19b043d2e1 100644 --- a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatRouterInterfaceListener.java +++ b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatRouterInterfaceListener.java @@ -21,11 +21,13 @@ import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase; import org.opendaylight.genius.infra.ManagedNewTransactionRunner; import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl; import org.opendaylight.genius.mdsalutil.MDSALUtil; +import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager; import org.opendaylight.infrautils.utils.concurrent.ListenableFutures; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterface; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterfaceBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterfaceKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterInterfacesMap; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.router.interfaces.map.RouterInterfaces; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.router.interfaces.map.router.interfaces.Interfaces; @@ -41,13 +43,21 @@ public class NatRouterInterfaceListener private final DataBroker dataBroker; private final ManagedNewTransactionRunner txRunner; private final OdlInterfaceRpcService interfaceManager; + private final IMdsalApiManager mdsalManager; + private final NaptManager naptManager; + private final NeutronvpnService neutronVpnService; @Inject - public NatRouterInterfaceListener(final DataBroker dataBroker, final OdlInterfaceRpcService interfaceManager) { + public NatRouterInterfaceListener(final DataBroker dataBroker, final OdlInterfaceRpcService interfaceManager, + final IMdsalApiManager mdsalManager,final NaptManager naptManager, + final NeutronvpnService neutronvpnService) { super(Interfaces.class, NatRouterInterfaceListener.class); this.dataBroker = dataBroker; this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker); this.interfaceManager = interfaceManager; + this.mdsalManager = mdsalManager; + this.naptManager = naptManager; + this.neutronVpnService = neutronvpnService; } @Override @@ -130,6 +140,10 @@ public class NatRouterInterfaceListener final ReentrantLock lock = NatUtil.lockForNat(dpId); lock.lock(); try { + if (NatUtil.isSnatEnabledForRouterId(dataBroker, routerId)) { + NatUtil.removeSnatEntriesForPort(dataBroker, naptManager, mdsalManager, neutronVpnService, + interfaceName, routerId); + } ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> { //Delete the NeutronRouterDpnMap from the ODL:L3VPN operational model NatUtil.removeFromNeutronRouterDpnsMap(routerId, interfaceName, dpId, operTx); diff --git a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatSouthboundEventHandlers.java b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatSouthboundEventHandlers.java index 6bcc99185b..881ff6d558 100644 --- a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatSouthboundEventHandlers.java +++ b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatSouthboundEventHandlers.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.locks.ReentrantLock; import javax.inject.Inject; import javax.inject.Singleton; @@ -35,30 +34,17 @@ import org.opendaylight.genius.infra.Datastore.Operational; import org.opendaylight.genius.infra.ManagedNewTransactionRunner; import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl; import org.opendaylight.genius.infra.TypedReadWriteTransaction; -import org.opendaylight.genius.mdsalutil.FlowEntity; -import org.opendaylight.genius.mdsalutil.NwConstants; import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager; import org.opendaylight.infrautils.jobcoordinator.JobCoordinator; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface; -import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.router.interfaces.RouterInterface; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.RouterPorts; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.Ports; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.neutron.vip.states.VipState; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.IpPort; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.ip.port.IntIpProtoType; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortInputBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; -import org.opendaylight.yangtools.yang.common.RpcResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -324,51 +310,6 @@ public class NatSouthboundEventHandlers { return port.get().nonnullInternalToExternalPortMap(); } - @Nullable - private BigInteger getNaptSwitchforRouter(DataBroker broker, String routerName) { - InstanceIdentifier rtrNaptSw = InstanceIdentifier.builder(NaptSwitches.class) - .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build(); - Optional routerToNaptSwitchData = - SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(broker, - LogicalDatastoreType.CONFIGURATION, rtrNaptSw); - if (routerToNaptSwitchData.isPresent()) { - RouterToNaptSwitch routerToNaptSwitchInstance = routerToNaptSwitchData.get(); - return routerToNaptSwitchInstance.getPrimarySwitchId(); - } - return null; - } - - private void removeNatFlow(BigInteger dpnId, short tableId, Long routerId, String ipAddress, int ipPort) { - - String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), ipAddress, ipPort); - FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef); - - mdsalManager.removeFlow(snatFlowEntity); - LOG.debug("removeNatFlow : Removed the flow in table {} for the switch with the DPN ID {} for " - + "router {} ip {} port {}", tableId, dpnId, routerId, ipAddress, ipPort); - } - - @Nullable - private List getFixedIpsForPort(String interfname) { - LOG.debug("getFixedIpsForPort : getFixedIpsForPort method is called for interface {}", interfname); - try { - Future> result = - neutronVpnService.getFixedIPsForNeutronPort(new GetFixedIPsForNeutronPortInputBuilder() - .setPortId(new Uuid(interfname)).build()); - - RpcResult rpcResult = result.get(); - if (!rpcResult.isSuccessful()) { - LOG.error("getFixedIpsForPort : RPC Call to GetFixedIPsForNeutronPortOutput returned with Errors {}", - rpcResult.getErrors()); - } else { - return rpcResult.getResult().getFixedIPs(); - } - } catch (InterruptedException | ExecutionException | NullPointerException ex) { - LOG.error("getFixedIpsForPort : Exception while receiving fixedIps for port {}", interfname, ex); - } - return null; - } - private void processInterfaceRemoved(String portName, BigInteger dpnId, String routerId, List> futures) { LOG.trace("processInterfaceRemoved : Processing Interface Removed Event for interface {} on DPN ID {}", @@ -393,89 +334,6 @@ public class NatSouthboundEventHandlers { } } - // TODO Clean up the exception handling - @SuppressWarnings("checkstyle:IllegalCatch") - private void removeSnatEntriesForPort(String interfaceName, String routerName) { - Long routerId = NatUtil.getVpnId(dataBroker, routerName); - if (routerId == NatConstants.INVALID_ID) { - LOG.error("removeSnatEntriesForPort : routerId not found for routername {}", routerName); - return; - } - BigInteger naptSwitch = getNaptSwitchforRouter(dataBroker, routerName); - if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) { - LOG.error("removeSnatEntriesForPort : NaptSwitch is not elected for router {} with Id {}", - routerName, routerId); - return; - } - //getInternalIp for port - List fixedIps = getFixedIpsForPort(interfaceName); - if (fixedIps == null) { - LOG.warn("removeSnatEntriesForPort : Internal Ips not found for InterfaceName {} in router {} with id {}", - interfaceName, routerName, routerId); - return; - } - - for (String internalIp : fixedIps) { - LOG.debug("removeSnatEntriesForPort : Internal Ip retrieved for interface {} is {} in router with Id {}", - interfaceName, internalIp, routerId); - IpPort ipPort = NatUtil.getInternalIpPortInfo(dataBroker, routerId, internalIp); - if (ipPort == null) { - LOG.debug("removeSnatEntriesForPort : no snatint-ip-port-map found for ip:{}", internalIp); - continue; - } - - for (IntIpProtoType protoType : ipPort.nonnullIntIpProtoType()) { - ProtocolTypes protocol = protoType.getProtocol(); - @Nullable List ports = protoType.getPorts(); - for (Integer portnum : (ports != null ? ports : Collections.emptyList())) { - //build and remove the flow in outbound table - try { - removeNatFlow(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, routerId, internalIp, portnum); - } catch (Exception ex) { - LOG.error("removeSnatEntriesForPort : Failed to remove snat flow for internalIP {} with " - + "Port {} protocol {} for routerId {} in OUTBOUNDTABLE of NaptSwitch {}", - internalIp, portnum, protocol, routerId, naptSwitch, ex); - } - //Get the external IP address and the port from the model - NAPTEntryEvent.Protocol proto = protocol.toString().equals(ProtocolTypes.TCP.toString()) - ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP; - IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId, - internalIp, String.valueOf(portnum), proto); - if (ipPortExternal == null) { - LOG.error("removeSnatEntriesForPort : Mapping for internalIp {} with port {} is not found in " - + "router with Id {}", internalIp, portnum, routerId); - return; - } - String externalIpAddress = ipPortExternal.getIpAddress(); - Integer portNumber = ipPortExternal.getPortNum(); - - //build and remove the flow in inboundtable - try { - removeNatFlow(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, routerId, - externalIpAddress, portNumber); - } catch (Exception ex) { - LOG.error("removeSnatEntriesForPort : Failed to remove snat flow internalIP {} with " - + "Port {} protocol {} for routerId {} in INBOUNDTABLE of naptSwitch {}", - externalIpAddress, portNumber, protocol, routerId, naptSwitch, ex); - } - - String internalIpPort = internalIp + ":" + portnum; - // delete the entry from IntExtIpPortMap DS - try { - naptManager.removeFromIpPortMapDS(routerId, internalIpPort, proto); - naptManager.removePortFromPool(internalIpPort, externalIpAddress); - } catch (Exception ex) { - LOG.error("removeSnatEntriesForPort : releaseIpExtPortMapping failed, Removal of " - + "ipportmap {} for router {} failed", internalIpPort, routerId, ex); - } - } - } - // delete the entry from SnatIntIpPortMap DS - LOG.debug("removeSnatEntriesForPort : Removing InternalIp:{} on router {}", internalIp, routerId); - naptManager.removeFromSnatIpPortDS(routerId, internalIp); - } - } - private class NatFlowAddWorker implements Callable>> { private final String interfaceName; private final String routerName; @@ -532,7 +390,10 @@ public class NatSouthboundEventHandlers { } else if (state.equals(IntfTransitionState.STATE_DOWN)) { LOG.debug("call : Port DOWN event received for interface {} ", interfaceName); try { - removeSnatEntriesForPort(interfaceName, routerName); + if (NatUtil.isSnatEnabledForRouterId(dataBroker, routerName)) { + NatUtil.removeSnatEntriesForPort(dataBroker, naptManager, mdsalManager, neutronVpnService, + interfaceName, routerName); + } } catch (Exception ex) { LOG.error("call : Exception caught in Interface {} OperationalStateDown", interfaceName, ex); } @@ -560,7 +421,10 @@ public class NatSouthboundEventHandlers { try { LOG.trace("call : Port removed event received for interface {} ", interfaceName); processInterfaceRemoved(interfaceName, intfDpnId, routerName, futures); - removeSnatEntriesForPort(interfaceName, routerName); + if (NatUtil.isSnatEnabledForRouterId(dataBroker, routerName)) { + NatUtil.removeSnatEntriesForPort(dataBroker, naptManager, mdsalManager, neutronVpnService, + interfaceName, routerName); + } } catch (Exception e) { LOG.error("call : Exception caught in Interface {} OperationalStateRemove", interfaceName, e); } diff --git a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatUtil.java b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatUtil.java index c53b27fb63..7b72c484ee 100644 --- a/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatUtil.java +++ b/natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NatUtil.java @@ -16,6 +16,7 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.base.Strings; +import com.google.common.collect.Iterables; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.math.BigInteger; import java.net.InetAddress; @@ -40,6 +41,7 @@ import org.opendaylight.controller.md.sal.binding.api.DataBroker; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException; +import org.opendaylight.controller.sal.common.util.Arguments; import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker; import org.opendaylight.genius.infra.Datastore.Configuration; import org.opendaylight.genius.infra.Datastore.Operational; @@ -131,6 +133,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces; @@ -231,8 +234,11 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev16011 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.IpPortKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.ip.port.IntIpProtoType; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.snatint.ip.port.map.intip.port.map.ip.port.IntIpProtoTypeKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortInputBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.GetFixedIPsForNeutronPortOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkMaps; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronVpnPortipPortData; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnService; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.VpnMaps; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.networkmaps.NetworkMap; @@ -260,6 +266,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.re import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.ovsdb.node.attributes.OpenvswitchOtherConfigs; import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; +import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.PathArgument; import org.opendaylight.yangtools.yang.common.RpcResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -394,19 +401,23 @@ public final class NatUtil { getFlowRef() returns a string identfier for the SNAT flows using the router ID as the reference. */ public static String getFlowRef(BigInteger dpnId, short tableId, long routerID, String ip) { - return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants - .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + ip; + return new StringBuilder().append(NatConstants.NAPT_FLOWID_PREFIX).append(dpnId) + .append(NatConstants.FLOWID_SEPARATOR).append(tableId).append(NatConstants.FLOWID_SEPARATOR) + .append(routerID).append(NatConstants.FLOWID_SEPARATOR).append(ip).toString(); } public static String getFlowRef(BigInteger dpnId, short tableId, InetAddress destPrefix, long vpnId) { - return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants - .FLOWID_SEPARATOR + destPrefix.getHostAddress() + NatConstants.FLOWID_SEPARATOR + vpnId; + return new StringBuilder().append(NatConstants.NAPT_FLOWID_PREFIX).append(dpnId) + .append(NatConstants.FLOWID_SEPARATOR).append(tableId).append(NatConstants.FLOWID_SEPARATOR) + .append(destPrefix.getHostAddress()).append(NatConstants.FLOWID_SEPARATOR).append(vpnId).toString(); } - public static String getNaptFlowRef(BigInteger dpnId, short tableId, String routerID, String ip, int port) { - return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants - .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + ip + NatConstants.FLOWID_SEPARATOR - + port; + public static String getNaptFlowRef(BigInteger dpnId, short tableId, String routerID, String ip, + int port, String protocol) { + return new StringBuilder().append(NatConstants.NAPT_FLOWID_PREFIX).append(dpnId) + .append(NatConstants.FLOWID_SEPARATOR).append(tableId).append(NatConstants.FLOWID_SEPARATOR) + .append(routerID).append(NatConstants.FLOWID_SEPARATOR).append(ip).append(NatConstants.FLOWID_SEPARATOR) + .append(port).append(NatConstants.FLOWID_SEPARATOR).append(protocol).toString(); } @Nullable @@ -2567,4 +2578,127 @@ public final class NatUtil { // FIXME: wrap this in an Identifier return JvmGlobalLocks.getLockForString(NatConstants.NAT_DJC_PREFIX + dataPath); } + + public static void removeSnatEntriesForPort(DataBroker dataBroker, NaptManager naptManager, + IMdsalApiManager mdsalManager, NeutronvpnService neutronVpnService, + String interfaceName, String routerName) { + Long routerId = NatUtil.getVpnId(dataBroker, routerName); + if (routerId == NatConstants.INVALID_ID) { + LOG.error("removeSnatEntriesForPort: routerId not found for routername {}", routerName); + return; + } + BigInteger naptSwitch = getPrimaryNaptfromRouterName(dataBroker, routerName); + if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) { + LOG.error("removeSnatEntriesForPort: NaptSwitch is not elected for router {}" + + "with Id {}", routerName, routerId); + return; + } + //getInternalIp for port + List fixedIps = getFixedIpsForPort(neutronVpnService, interfaceName); + if (fixedIps == null) { + LOG.error("removeSnatEntriesForPort: Internal Ips not found for InterfaceName {} in router {} with id {}", + interfaceName, routerName, routerId); + return; + } + List protocolTypesList = getPortocolList(); + for (String internalIp : fixedIps) { + LOG.debug("removeSnatEntriesForPort: Internal Ip retrieved for interface {} is {} in router with Id {}", + interfaceName, internalIp, routerId); + for (ProtocolTypes protocol : protocolTypesList) { + List portList = NatUtil.getInternalIpPortListInfo(dataBroker, routerId, internalIp, protocol); + if (portList != null) { + for (Integer portnum : portList) { + //build and remove the flow in outbound table + removeNatFlow(mdsalManager, naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, + routerId, internalIp, portnum, protocol.getName()); + + //build and remove the flow in inboundtable + + removeNatFlow(mdsalManager, naptSwitch, NwConstants.INBOUND_NAPT_TABLE, routerId, + internalIp, portnum, protocol.getName()); + + //Get the external IP address and the port from the model + + NAPTEntryEvent.Protocol proto = protocol.toString().equals(ProtocolTypes.TCP.toString()) + ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP; + IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId, + internalIp, String.valueOf(portnum), proto); + if (ipPortExternal == null) { + LOG.error("removeSnatEntriesForPort: Mapping for internalIp {} " + + "with port {} is not found in " + + "router with Id {}", internalIp, portnum, routerId); + return; + } + String externalIpAddress = ipPortExternal.getIpAddress(); + String internalIpPort = internalIp + ":" + portnum; + // delete the entry from IntExtIpPortMap DS + + naptManager.removeFromIpPortMapDS(routerId, internalIpPort, proto); + naptManager.removePortFromPool(internalIpPort, externalIpAddress); + + } + } else { + LOG.debug("removeSnatEntriesForPort: No {} session for interface {} with internalIP {} " + + "in router with id {}", + protocol, interfaceName, internalIp, routerId); + } + } + // delete the entry from SnatIntIpPortMap DS + LOG.debug("removeSnatEntriesForPort: Removing InternalIp :{} of router {} from snatint-ip-port-map", + internalIp, routerId); + naptManager.removeFromSnatIpPortDS(routerId, internalIp); + } + } + + private static List getFixedIpsForPort(NeutronvpnService neutronVpnService, String interfname) { + LOG.debug("getFixedIpsForPort: getFixedIpsForPort method is called for interface {}", interfname); + try { + Future> result = + neutronVpnService.getFixedIPsForNeutronPort(new GetFixedIPsForNeutronPortInputBuilder() + .setPortId(new Uuid(interfname)).build()); + + RpcResult rpcResult = result.get(); + if (!rpcResult.isSuccessful()) { + LOG.error("getFixedIpsForPort: RPC Call to GetFixedIPsForNeutronPortOutput returned with Errors {}", + rpcResult.getErrors()); + } else { + return rpcResult.getResult().getFixedIPs(); + } + } catch (InterruptedException | ExecutionException | NullPointerException ex) { + LOG.error("getFixedIpsForPort: Exception while receiving fixedIps for port {}", interfname, ex); + } + return null; + } + + private static List getPortocolList() { + List protocollist = new ArrayList<>(); + protocollist.add(ProtocolTypes.TCP); + protocollist.add(ProtocolTypes.UDP); + return protocollist; + } + + private static void removeNatFlow(IMdsalApiManager mdsalManager, BigInteger dpnId, short tableId, Long routerId, + String ipAddress, int ipPort, String protocol) { + + String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, tableId, String.valueOf(routerId), ipAddress, ipPort, + protocol); + FlowEntity snatFlowEntity = NatUtil.buildFlowEntity(dpnId, tableId, switchFlowRef); + + mdsalManager.removeFlow(snatFlowEntity); + LOG.debug("removeNatFlow: Removed the flow in table {} for the switch with the DPN ID {} for " + + "router {} ip {} port {}", tableId, dpnId, routerId, ipAddress, ipPort); + } + + public static String getDpnFromNodeRef(NodeRef node) { + PathArgument pathArgument = Iterables.get(node.getValue().getPathArguments(), 1); + InstanceIdentifier.IdentifiableItem item = Arguments.checkInstanceOf(pathArgument, + InstanceIdentifier.IdentifiableItem.class); + NodeKey key = Arguments.checkInstanceOf(item.getKey(), NodeKey.class); + String dpnKey = key.getId().getValue(); + String dpnID = null; + if (dpnKey.contains(NatConstants.COLON_SEPARATOR)) { + dpnID = new BigInteger(dpnKey.split(NatConstants.COLON_SEPARATOR)[1]).toString(); + } + return dpnID; + } } -- 2.36.6