Bug-2094 : L3 North-South does not work -- fix ARP table
authorFlavio Fernandes <ffernand@redhat.com>
Tue, 16 Jun 2015 00:42:36 +0000 (20:42 -0400)
committerFlavio Fernandes <ffernand@redhat.com>
Fri, 19 Jun 2015 00:39:39 +0000 (20:39 -0400)
This set of changes will fix the ARP responder table (ie table 20) to properly
handle the responding of ARP queries made on br-ex for the associated floating
ip addresses.

Change-Id: I72c5f24d00a42fb011441849adfdb244ba01937b
Signed-off-by: Flavio Fernandes <ffernand@redhat.com>
openstack/net-virt-providers/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/providers/openflow13/services/ArpResponderService.java
openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/SouthboundHandler.java
openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java
utils/mdsal-openflow/src/main/java/org/opendaylight/ovsdb/utils/mdsal/openflow/MatchUtils.java

index 250e03f4e36ae007b3f482d35cbe53efd2b951dc..83de1d1d7d7e0076716de2aa25d7c695beb43dce 100644 (file)
@@ -73,7 +73,15 @@ public class ArpResponderService extends AbstractServiceInstance implements ArpP
         ActionBuilder ab = new ActionBuilder();
         List<org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action> actionList = Lists.newArrayList();
 
-        MatchUtils.createTunnelIDMatch(matchBuilder, new BigInteger(segmentationId));
+        if (segmentationId != null) {
+            final Long inPort = MatchUtils.parseExplicitOFPort(segmentationId);
+            if (inPort != null) {
+                MatchUtils.createInPortMatch(matchBuilder, dpid, inPort);
+            } else {
+                MatchUtils.createTunnelIDMatch(matchBuilder, new BigInteger(segmentationId));
+            }
+        }
+
         MatchUtils.createEtherTypeMatch(matchBuilder, new EtherType(Constants.ARP_ETHERTYPE));
         MatchUtils.createArpDstIpv4Match(matchBuilder, MatchUtils.iPv4PrefixFromIPv4Address(ipAddress.getHostAddress()));
 
index f7a27322af569c253cc9fc8c6c17aa46b7f5b659..739dbe22c9ea07f7bca86fdb4e58c5c1e1739fb1 100644 (file)
@@ -82,13 +82,13 @@ public class SouthboundHandler extends AbstractHandler
         NeutronNetwork network = tenantNetworkManager.getTenantNetwork(tp);
         if (network != null && !network.getRouterExternal()) {
             logger.trace("handleInterfaceUpdate <{}> <{}> network: {}", node, tp, network.getNetworkUUID());
-            neutronL3Adapter.handleInterfaceEvent(node, tp, network, Action.UPDATE);
             if (bridgeConfigurationManager.createLocalNetwork(node, network)) {
                 networkingProviderManager.getProvider(node).handleInterfaceUpdate(network, node, tp);
             }
         } else {
             logger.debug("No tenant network found on node: <{}> for interface: <{}>", node, tp);
         }
+        neutronL3Adapter.handleInterfaceEvent(node, tp, network, Action.UPDATE);
     }
 
     private void handleInterfaceDelete (Node node, OvsdbTerminationPointAugmentation intf,
index f6d0a1f5054316897b4e81faf387320fb1d371c6..7560e0a43837d2e382f199b869084a357a08f65e 100644 (file)
@@ -9,6 +9,8 @@
  */
 package org.opendaylight.ovsdb.openstack.netvirt.impl;
 
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
 import org.opendaylight.neutron.spi.INeutronPortCRUD;
 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
@@ -22,6 +24,7 @@ import org.opendaylight.neutron.spi.Neutron_IPs;
 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
 
@@ -35,6 +38,7 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -60,20 +64,46 @@ public class NeutronL3Adapter implements ConfigInterface {
     private volatile ArpProvider arpProvider;
     private volatile RoutingProvider routingProvider;
 
+    private class FloatIpData {
+        FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
+                    final String floatingIpAddress, final String fixedIpAddress) {
+            this.dpid = dpid;
+            this.ofPort = ofPort;
+            this.segId = segId;
+            this.macAddress = macAddress;
+            this.floatingIpAddress = floatingIpAddress;
+            this.fixedIpAddress = fixedIpAddress;
+        }
+
+        public final Long dpid;
+        public final Long ofPort;
+        public final String segId;
+        public final String macAddress;
+        public final String floatingIpAddress;
+        public final String fixedIpAddress;
+    }
+
     private Set<String> inboundIpRewriteCache;
     private Set<String> outboundIpRewriteCache;
-    private Set<String> inboundIpRewriteExclusionCache;
     private Set<String> outboundIpRewriteExclusionCache;
     private Set<String> routerInterfacesCache;
     private Set<String> staticArpEntryCache;
     private Set<String> l3ForwardingCache;
     private Set<String> defaultRouteCache;
     private Map<String, String> networkIdToRouterMacCache;
+    private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
+    private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
+    private Map<String, FloatIpData> floatIpDataMapCache;
     private Boolean enabled = false;
     private Boolean flgDistributedARPEnabled = true;
     private Southbound southbound;
 
+    final static String OWNER_ROUTER_INTERFACE = "network:router_interface";
+    final static String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
+    final static String OWNER_ROUTER_GATEWAY = "network:router_gateway";
+    final static String OWNER_FLOATING_IP = "network:floatingip";
+
     public NeutronL3Adapter() {
         logger.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
     }
@@ -84,14 +114,17 @@ public class NeutronL3Adapter implements ConfigInterface {
         if (configurationService.isL3ForwardingEnabled()) {
             this.inboundIpRewriteCache = new HashSet<>();
             this.outboundIpRewriteCache = new HashSet<>();
-            this.inboundIpRewriteExclusionCache = new HashSet<>();
             this.outboundIpRewriteExclusionCache = new HashSet<>();
             this.routerInterfacesCache = new HashSet<>();
             this.staticArpEntryCache = new HashSet<>();
             this.l3ForwardingCache = new HashSet<>();
             this.defaultRouteCache = new HashSet<>();
             this.networkIdToRouterMacCache = new HashMap<>();
+            this.networkIdToRouterIpListCache = new HashMap<>();
             this.subnetIdToRouterInterfaceCache = new HashMap<>();
+            this.neutronPortToDpIdCache = new HashMap<>();
+            this.floatIpDataMapCache = new HashMap<>();
+
             this.enabled = true;
             logger.info("OVSDB L3 forwarding is enabled");
             if (configurationService.isDistributedArpDisabled()) {
@@ -125,7 +158,8 @@ public class NeutronL3Adapter implements ConfigInterface {
         // Treat the port event as a router interface event if the port belongs to router. This is a
         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
         //
-        if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
+        if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
+            neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
                 NeutronRouter_Interface neutronRouterInterface =
                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
@@ -193,14 +227,149 @@ public class NeutronL3Adapter implements ConfigInterface {
 
     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
                                              Action action) {
+        Preconditions.checkNotNull(neutronFloatingIP);
+
         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
-                     neutronFloatingIP.getFixedIPAddress(),
-                     neutronFloatingIP.getFloatingIPAddress(),
-                     neutronFloatingIP.getFloatingNetworkUUID());
+                neutronFloatingIP.getFixedIPAddress(),
+                neutronFloatingIP.getFloatingIPAddress(),
+                neutronFloatingIP.getFloatingNetworkUUID());
         if (!this.enabled)
             return;
 
-        this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
+        // Consider action to be delete if getFixedIPAddress is null
+        //
+        if (neutronFloatingIP.getFixedIPAddress() == null) {
+            action = Action.DELETE;
+        }
+
+        // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
+
+        if (action != Action.DELETE) {
+            programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
+
+            programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
+        } else {
+            programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
+
+            programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
+        }
+    }
+
+    private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
+        Preconditions.checkNotNull(neutronFloatingIP);
+
+        final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
+        if (fid == null) {
+            logger.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
+                    action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
+            return;
+        }
+        programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
+                                      action);
+    }
+
+    private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
+        Preconditions.checkNotNull(neutronFloatingIP);
+        Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
+        Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
+
+        if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
+            logger.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
+                    neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
+            return;
+        }
+
+        // find bridge Node where floating ip is configured by looking up cache for its port
+        final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
+        final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
+        final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
+        final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
+        final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
+        final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
+
+        final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
+        final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
+                neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
+        final String providerSegmentationId = tenantNeutronNetwork != null ?
+                tenantNeutronNetwork.getProviderSegmentationID() : null;
+
+        if (nodeIfPair == null || neutronTenantPortUuid == null ||
+                providerSegmentationId == null || providerSegmentationId.isEmpty() ||
+                floatingIpMac == null || floatingIpMac.isEmpty()) {
+            logger.trace("Floating IP {}<->{}, found no dpid for floatPort {} tenantPortUuid {} seg {} mac {}",
+                    fixedIpAddress,
+                    floatingIpAddress,
+                    neutronPortForFloatIp,
+                    neutronTenantPortUuid,
+                    providerSegmentationId,
+                    floatingIpMac);
+            return;
+        }
+
+        // get ofport for patch port in br-int
+        final Long dpId = nodeIfPair.getLeft();
+        final Long ofPort = findOFPortForExtPatch(dpId);
+        if (ofPort == null) {
+            logger.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
+                    dpId);
+            return;
+        }
+
+        // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
+        //
+        if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
+                Action.ADD)) {
+            final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
+                    floatingIpAddress, fixedIpAddress);
+            floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
+            logger.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
+                    neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
+                    floatingIpMac, ofPort, providerSegmentationId, dpId);
+        }
+    }
+
+    private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
+        final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
+        if (floatIpData == null) {
+            logger.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
+            return;
+        }
+
+        if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
+                floatIpData.floatingIpAddress, Action.DELETE)) {
+            floatIpDataMapCache.remove(neutronFloatingIPUuid);
+            logger.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
+                    floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
+        }
+    }
+
+    private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
+        for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
+            if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
+                    neutronPort.getDeviceID().equals(floatingIpUuid)) {
+                return neutronPort;
+            }
+        }
+        return null;
+    }
+
+    private final Long findOFPortForExtPatch(Long dpId) {
+        final String brInt = configurationService.getIntegrationBridgeName();
+        final String brExt = configurationService.getExternalBridgeName();
+        final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
+
+        Preconditions.checkNotNull(dpId);
+        Preconditions.checkNotNull(portNameInt);
+
+        final long dpidPrimitive = dpId.longValue();
+        for (Node node : nodeCacheManager.getBridgeNodes()) {
+            if (dpidPrimitive == southbound.getDataPathId(node)) {
+                final OvsdbTerminationPointAugmentation terminationPointOfBridge =
+                        southbound.getTerminationPointOfBridge(node, portNameInt);
+                return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
+            }
+        }
+        return null;
     }
 
     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
@@ -212,16 +381,53 @@ public class NeutronL3Adapter implements ConfigInterface {
     //
     // Callbacks from OVSDB's southbound handler
     //
-    public void handleInterfaceEvent(final Node node, final OvsdbTerminationPointAugmentation intf,
+    public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
                                      final NeutronNetwork neutronNetwork, Action action) {
         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
-                     action, node, intf.getName(), neutronNetwork);
-        if (!this.enabled)
+                     action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
+        if (!this.enabled) {
             return;
+        }
+
+        final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
+        final Long dpId = getDpidForIntegrationBridge(bridgeNode);
+        final Uuid interfaceUuid = intf.getInterfaceUuid();
+
+        logger.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
+                action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 
-        NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
         if (neutronPort != null) {
-            this.handleNeutronPortEvent(neutronPort, action);
+            final String neutronPortUuid = neutronPort.getPortUUID();
+
+            if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
+                    dpId != null && interfaceUuid != null) {
+                handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
+            }
+
+            handleNeutronPortEvent(neutronPort, action);
+        }
+
+        if (action == Action.DELETE && interfaceUuid != null) {
+            handleInterfaceEventDelete(intf, dpId);
+        }
+    }
+
+    private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
+        neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
+        logger.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
+                neutronPortUuid, dpId, interfaceUuid.getValue());
+    }
+
+    private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
+        // Remove entry from neutronPortToDpIdCache based on interface uuid
+        for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
+            final String currPortUuid = entry.getKey();
+            if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
+                logger.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
+                        currPortUuid, dpId, intf.getInterfaceUuid().getValue());
+                neutronPortToDpIdCache.remove(currPortUuid);
+                break;
+            }
         }
     }
 
@@ -265,8 +471,6 @@ public class NeutronL3Adapter implements ConfigInterface {
                 continue;
             }
 
-            final boolean tenantNetworkPresentInNode =
-                    tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
                 final String tenantIpStr = neutronIP.getIpAddress();
                 if (tenantIpStr.isEmpty()) {
@@ -277,10 +481,9 @@ public class NeutronL3Adapter implements ConfigInterface {
                 // still needed when routing to subnets non-local to node (bug 2076).
                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 
-                // Configure distributed ARP responder. Only needed if tenant network exists in node.
+                // Configure distributed ARP responder
                 if (true == flgDistributedARPEnabled) {
-                    programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
-                                       tenantNetworkPresentInNode ? action : Action.DELETE);
+                    programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
                 }
             }
         }
@@ -336,10 +539,10 @@ public class NeutronL3Adapter implements ConfigInterface {
         if (status.isSuccess()) {
             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
                          l3ForwardingProvider == null ? "skipped" : "programmed",
-                         macAddress, address, node, actionForNode);
+                         macAddress, address, node.getNodeId().getValue(), actionForNode);
         } else {
             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
-                         macAddress, address, node, actionForNode, status);
+                         macAddress, address, node.getNodeId().getValue(), actionForNode, status);
         }
         return status;
     }
@@ -351,8 +554,8 @@ public class NeutronL3Adapter implements ConfigInterface {
         Preconditions.checkNotNull(destNeutronRouterInterface);
 
         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
-        final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
-        final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
+        String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
+        List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
         final NeutronNetwork neutronNetwork = subnet != null ?
                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
@@ -366,6 +569,17 @@ public class NeutronL3Adapter implements ConfigInterface {
         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
                      destNeutronRouterInterface, isDelete);
 
+        // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
+        // the local cache
+        if (neutronNetwork != null) {
+            if (macAddress == null || macAddress.isEmpty()) {
+                macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
+            }
+            if (ipList == null || ipList.isEmpty()) {
+                ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
+            }
+        }
+
         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
             cidr == null || cidr.isEmpty() ||
             macAddress == null || macAddress.isEmpty() ||
@@ -381,6 +595,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         //
         if (! isDelete) {
             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
+            networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
         }
 
@@ -412,8 +627,12 @@ public class NeutronL3Adapter implements ConfigInterface {
                                                               true /*isReflexsive*/);
                 }
 
-                    // Enable ARP responder by default, because router interface needs to be responded always.
-                    programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
+                if (! isExternal) {
+                    programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
+                            actionForNode);
+                }
+                // Enable ARP responder by default, because router interface needs to be responded always.
+                programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
             }
 
             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
@@ -421,10 +640,7 @@ public class NeutronL3Adapter implements ConfigInterface {
             //
             {
                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
-                programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
-                                                cidr, actionForRewriteExclusion);
-                programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
-                                                cidr, actionForRewriteExclusion);
+                programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
             }
 
             // Default route. For non-external subnet, make sure that there is none configured.
@@ -442,10 +658,22 @@ public class NeutronL3Adapter implements ConfigInterface {
         //
         if (isDelete) {
             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
+            networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
         }
     }
 
+    private void programFlowForNetworkFromExternal(final Node node,
+                                                   final Long dpid,
+                                                   final String destinationSegmentationId,
+                                                   final String dstMacAddress,
+                                                   final String destIpStr,
+                                                   final int destMask,
+                                                   final Action actionForNode) {
+        programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
+                dstMacAddress, destIpStr, destMask, actionForNode);
+    }
+
     private void programFlowsForNeutronRouterInterfacePair(final Node node,
                                                            final Long dpid,
                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
@@ -587,139 +815,186 @@ public class NeutronL3Adapter implements ConfigInterface {
         if (status.isSuccess()) {
             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
                          routingProvider == null ? "skipped" : "programmed",
-                         macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
+                         macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
                          actionForNode);
         } else {
             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
-                         macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
+                         macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
                          actionForNode, status);
         }
         return status;
     }
 
-    private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
-                                        String macAddress, String ipStr,
-                                        Action actionForNode) {
+    private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
+                                           String macAddress, String ipStr,
+                                           Action action) {
         // Based on the local cache, figure out whether programming needs to occur. To do this, we
         // will look at desired action for node.
         //
-        final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
+        final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 
-        if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
-                    node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
-            return;
+        if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
+            logger.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
+                    dpid, segOrOfPort, macAddress, ipStr, action);
+            return true;
         }
-        if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
-            logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
-                    node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
-            return;
+        if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
+            logger.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
+                    dpid, segOrOfPort, macAddress, ipStr, action);
+            return true;
         }
 
-        Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
-                                                    macAddress, ipStr, actionForNode);
+        Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
         if (status.isSuccess()) {
             // Update cache
-            if (actionForNode == Action.ADD) {
+            if (action == Action.ADD) {
                 staticArpEntryCache.add(cacheKey);
             } else {
                 staticArpEntryCache.remove(cacheKey);
             }
+            return true;
         }
+        return false;
     }
 
-    private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
-                                                String macAddress,
-                                                String address,
-                                                Action actionForNode) {
+    private Status programStaticArpStage2(Long dpid,
+                                          String segOrOfPort,
+                                          String macAddress,
+                                          String address,
+                                          Action action) {
         Status status;
         try {
             InetAddress inetAddress = InetAddress.getByName(address);
             status = arpProvider == null ?
                      new Status(StatusCode.SUCCESS) :
-                     arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
-                                                       macAddress, inetAddress, actionForNode);
+                     arpProvider.programStaticArpEntry(dpid, segOrOfPort,
+                                                       macAddress, inetAddress, action);
         } catch (UnknownHostException e) {
             status = new Status(StatusCode.BADREQUEST);
         }
 
         if (status.isSuccess()) {
-            logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
+            logger.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
                          arpProvider == null ? "skipped" : "programmed",
-                         macAddress, address, node, actionForNode);
+                         macAddress, address, dpid, segOrOfPort, action);
         } else {
-            logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
-                         macAddress, address, node, actionForNode, status);
+            logger.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
+                         macAddress, address, dpid, segOrOfPort, action, status);
         }
         return status;
     }
 
-    private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
-                                                 final boolean isInbound, String cidr,
+    private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
+                                                  String matchAddress, String rewriteAddress,
+                                                  Action action) {
+        // Based on the local cache, figure out whether programming needs to occur. To do this, we
+        // will look at desired action for node.
+        //
+        final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
+        final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
+
+        if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
+            logger.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
+                    " action {} is already done",
+                    dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
+            return true;
+        }
+        if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
+            logger.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
+                    " action is already done",
+                    dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
+            return true;
+        }
+
+        Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
+                rewriteAddress, action);
+        if (status.isSuccess()) {
+            // Update cache
+            if (action == Action.ADD) {
+                inboundIpRewriteCache.add(cacheKey);
+            } else {
+                inboundIpRewriteCache.remove(cacheKey);
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
+                                                 String matchAddress, String rewriteAddress,
+                                                 Action action) {
+        Status status;
+        try {
+            InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
+            InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
+            status = inboundNatProvider == null ?
+                    new Status(StatusCode.SUCCESS) :
+                    inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
+                            inetMatchAddress, inetRewriteAddress,
+                            action);
+        } catch (UnknownHostException e) {
+            status = new Status(StatusCode.BADREQUEST);
+        }
+
+        if (status.isSuccess()) {
+            final boolean isSkipped = inboundNatProvider == null;
+            logger.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
+                    (isSkipped ? "skipped" : "programmed"),
+                    dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
+        } else {
+            logger.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
+                         " status:{}",
+                    dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
+                    status);
+        }
+        return status;
+    }
+
+    private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
                                                  Action actionForRewriteExclusion) {
         // Based on the local cache, figure out whether programming needs to occur. To do this, we
         // will look at desired action for node.
         //
         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
-        final Boolean isProgrammed = isInbound ?
-                                     inboundIpRewriteExclusionCache.contains(cacheKey):
-                                     outboundIpRewriteExclusionCache.contains(cacheKey);
+        final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
 
         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
-                         node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
-                         actionForRewriteExclusion);
+            logger.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
+                         node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
             return;
         }
         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
-            logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
-                         node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
-                         actionForRewriteExclusion);
+            logger.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
+                         node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
             return;
         }
 
         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
-                                                             isInbound, actionForRewriteExclusion);
+                                                             actionForRewriteExclusion);
         if (status.isSuccess()) {
             // Update cache
             if (actionForRewriteExclusion == Action.ADD) {
-                if (isInbound) {
-                    inboundIpRewriteExclusionCache.add(cacheKey);
-                } else {
                     outboundIpRewriteExclusionCache.add(cacheKey);
-                }
             } else {
-                if (isInbound) {
-                    inboundIpRewriteExclusionCache.remove(cacheKey);
-                } else {
                     outboundIpRewriteExclusionCache.remove(cacheKey);
-                }
             }
         }
     }
 
     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
-                                                   final boolean isInbound, Action actionForNode) {
-        Status status;
-        if (isInbound) {
-            status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
-                     inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
-                                                                  actionForNode);
-        } else {
-            status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
-                     outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
-                                                                   actionForNode);
-        }
+                                                   Action actionForNode) {
+        final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
+                outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
 
         if (status.isSuccess()) {
-            final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
-            logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
-                         (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
-                         cidr, node, actionForNode);
+            final boolean isSkipped = outboundNatProvider == null;
+            logger.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
+                         (isSkipped ? "skipped" : "programmed"),
+                         cidr, node.getNodeId().getValue(), actionForNode);
         } else {
-            logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
-                         (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
+            logger.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
+                         cidr, node.getNodeId().getValue(), actionForNode, status);
         }
         return status;
     }
@@ -765,7 +1040,7 @@ public class NeutronL3Adapter implements ConfigInterface {
         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
         if (defaultGatewayMacAddress == null) {
             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
-                         gatewayIp, node, actionForNodeDefaultRoute);
+                         gatewayIp, node.getNodeId().getValue(), actionForNodeDefaultRoute);
             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
         }
 
@@ -784,10 +1059,10 @@ public class NeutronL3Adapter implements ConfigInterface {
         if (status.isSuccess()) {
             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
                          routingProvider == null ? "skipped" : "programmed",
-                         defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
+                         defaultGatewayMacAddress, gatewayIp, node.getNodeId().getValue(), actionForNodeDefaultRoute);
         } else {
             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
-                         defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
+                         defaultGatewayMacAddress, gatewayIp, node.getNodeId().getValue(), actionForNodeDefaultRoute, status);
         }
         return status;
     }
@@ -832,20 +1107,11 @@ public class NeutronL3Adapter implements ConfigInterface {
                     action : Action.DELETE;
 
             // Rewrite from float to fixed and vice-versa
-            //
-            programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
-                                   floatingIpAddress, fixedIPAddress, actionForNode);
-            programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
-                                   fixedIPAddress, floatingIpAddress, actionForNode);
-
-            // Respond to ARPs for the floating ip address by default
-            programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
-                                   actionForNode);
+            programIpRewriteStage1(node, dpid, providerSegmentationId, fixedIPAddress, floatingIpAddress, actionForNode);
         }
     }
 
     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
-                                        final boolean isInbound,
                                         String matchAddress, String rewriteAddress,
                                         Action actionForNode) {
         // Based on the local cache, figure out whether programming needs to occur. To do this, we
@@ -853,77 +1119,58 @@ public class NeutronL3Adapter implements ConfigInterface {
         //
         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" +
                                 matchAddress + ":" + rewriteAddress;
-        final Boolean isProgrammed = isInbound ?
-                                     inboundIpRewriteCache.contains(cacheKey) :
-                                     outboundIpRewriteCache.contains(cacheKey);
+        final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
 
         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
-            logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
+            logger.trace("programIpRewriteStage1 node {} providerId {} matchAddr {} rewriteAddr {} action {}" +
                          " is already done",
-                         node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
-                         matchAddress, rewriteAddress, actionForNode);
+                         node.getNodeId().getValue(), providerSegmentationId, matchAddress, rewriteAddress,
+                    actionForNode);
             return;
         }
         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
-            logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
+            logger.trace("programIpRewriteStage1 node {} providerId {} matchAddr {} rewriteAddr {} action {}" +
                          " is already done",
-                         node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
-                         matchAddress, rewriteAddress, actionForNode);
+                         node.getNodeId().getValue(), providerSegmentationId, matchAddress, rewriteAddress,
+                         actionForNode);
             return;
         }
 
-        Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
-                                                    matchAddress, rewriteAddress, actionForNode);
+        Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, matchAddress, rewriteAddress,
+                                                    actionForNode);
         if (status.isSuccess()) {
             // Update cache
             if (actionForNode == Action.ADD) {
-                if (isInbound) {
-                    inboundIpRewriteCache.add(cacheKey);
-                } else {
-                    outboundIpRewriteCache.add(cacheKey);
-                }
+                outboundIpRewriteCache.add(cacheKey);
             } else {
-                if (isInbound) {
-                    inboundIpRewriteCache.remove(cacheKey);
-                } else {
-                    outboundIpRewriteCache.remove(cacheKey);
-                }
+                outboundIpRewriteCache.remove(cacheKey);
             }
         }
     }
 
     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
-                                          final boolean isInbound,
                                           String matchAddress, String rewriteAddress,
                                           Action actionForNode) {
         Status status;
         try {
             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
-            if (isInbound) {
-                status = inboundNatProvider == null ?
-                         new Status(StatusCode.SUCCESS) :
-                         inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
-                                                                 inetMatchAddress, inetRewriteAddress, actionForNode);
-            } else {
-                status = outboundNatProvider == null ?
-                         new Status(StatusCode.SUCCESS) :
-                         outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
-                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
-            }
+            status = outboundNatProvider == null ?
+                    new Status(StatusCode.SUCCESS) :
+                    outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
+                                                             inetMatchAddress, inetRewriteAddress, actionForNode);
         } catch (UnknownHostException e) {
             status = new Status(StatusCode.BADREQUEST);
         }
 
         if (status.isSuccess()) {
-            final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
-            logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
-                         (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
-                         matchAddress, rewriteAddress, node, actionForNode);
+            final boolean isSkipped = outboundNatProvider == null;
+            logger.debug("ProgramIpRewrite {} for match:{} rewrite:{} node:{} action:{}",
+                         (isSkipped ? "skipped" : "programmed"),
+                         matchAddress, rewriteAddress, node.getNodeId().getValue(), actionForNode);
         } else {
-            logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
-                         (isInbound ? "inbound" : "outbound"),
-                         matchAddress, rewriteAddress, node, actionForNode, status);
+            logger.error("ProgramIpRewrite failed for match:{} rewrite:{} node:{} action:{} status:{}",
+                         matchAddress, rewriteAddress, node.getNodeId().getValue(), actionForNode, status);
         }
         return status;
     }
@@ -952,6 +1199,16 @@ public class NeutronL3Adapter implements ConfigInterface {
         return null;
     }
 
+    /**
+     * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
+     *
+     * @param ofPort the OF port number
+     * @return the string with encoded OF port (example format "OFPort|999")
+     */
+    public static String encodeExcplicitOFPort(Long ofPort) {
+        return "OFPort|" + ofPort.toString();
+    }
+
     @Override
     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
         tenantNetworkManager =
index 47c26e395e93fd645438efd6f9c361facdaa8968..5e77ed06d2ca8e099568177652a12c9b2d3e29d8 100644 (file)
@@ -999,4 +999,20 @@ public class MatchUtils {
     public static Ipv4Prefix iPv4PrefixFromIPv4Address(String ipv4AddressString) {
         return new Ipv4Prefix(ipv4AddressString + "/32");
     }
+
+    /**
+     * Return Long that represents OF port for strings where OF is explicitly provided
+     *
+     * @param ofPortIdentifier the string with encoded OF port (example format "OFPort|999")
+     * @return the OFport or null
+     */
+    public static Long parseExplicitOFPort(String ofPortIdentifier) {
+        if (ofPortIdentifier != null) {
+            String[] pair = ofPortIdentifier.split("\\|");
+            if ((pair.length > 1) && (pair[0].equalsIgnoreCase("OFPort"))) {
+                return new Long(pair[1]);
+            }
+        }
+        return null;
+    }
 }