Bug 3378 - ovsdb netvirt needs help in getting mac for a given ip in br-ex 05/25205/1
authorAnil Vishnoi <vishnoianil@gmail.com>
Sun, 19 Jul 2015 01:58:31 +0000 (18:58 -0700)
committerAnil Vishnoi <vishnoianil@gmail.com>
Wed, 12 Aug 2015 14:37:33 +0000 (20:07 +0530)
Added periodic refresh of gateway mac address
Fixed packet deserialization exception

Change-Id: Ib898347770cee213883c5222649b267efb4c701a
Signed-off-by: Anil Vishnoi <vishnoianil@gmail.com>
openstack/net-virt-providers/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/providers/openflow13/services/arp/ArpResolverMetadata.java
openstack/net-virt-providers/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/providers/openflow13/services/arp/ArpResolverUtils.java
openstack/net-virt-providers/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/providers/openflow13/services/arp/GatewayMacResolverService.java
openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/api/GatewayMacResolver.java
openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

index 8947cdfb6f53cb7e90fa2a001bd9389df5454061..0ac0bc0884650f8d253a9b110e7b7069828575d6 100644 (file)
@@ -19,24 +19,21 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.service.rev130819.Remo
 
 public final class ArpResolverMetadata {
 
+    private final Ipv4Address gatewayIpAddress;
+    private final Long externalNetworkBridgeDpid;
+    private final Ipv4Address arpRequestSourceIp;
+    private final MacAddress arpRequestSourceMacAddress;
+    private final boolean periodicRefresh;
     private RemoveFlowInput flowToRemove;
-    private Ipv4Address gatewayIpAddress;
     private MacAddress gatewayMacAddress;
-    private boolean periodicRefresh;
 
-    public ArpResolverMetadata(){
-
-    }
-    public ArpResolverMetadata(RemoveFlowInput flowToRemove, Ipv4Address gatewayIpAddress, boolean periodicRefresh){
-        this.flowToRemove = flowToRemove;
+    public ArpResolverMetadata(final Long externalNetworkBridgeDpid,
+            final Ipv4Address gatewayIpAddress, final Ipv4Address arpRequestSourceIp,
+            final MacAddress arpRequestMacAddress, final boolean periodicRefresh){
+        this.externalNetworkBridgeDpid = externalNetworkBridgeDpid;
         this.gatewayIpAddress = gatewayIpAddress;
-        this.periodicRefresh = periodicRefresh;
-    }
-
-    public ArpResolverMetadata(RemoveFlowInput flowToRemove, Ipv4Address gatewayIpAddress, MacAddress gatewayMacAddress, boolean periodicRefresh){
-        this.flowToRemove = flowToRemove;
-        this.gatewayIpAddress = gatewayIpAddress;
-        this.gatewayMacAddress = gatewayMacAddress;
+        this.arpRequestSourceIp = arpRequestSourceIp;
+        this.arpRequestSourceMacAddress = arpRequestMacAddress;
         this.periodicRefresh = periodicRefresh;
     }
 
@@ -46,18 +43,12 @@ public final class ArpResolverMetadata {
     public boolean isPeriodicRefresh() {
         return periodicRefresh;
     }
-    public void setPeriodicRefresh(boolean periodicRefresh) {
-        this.periodicRefresh = periodicRefresh;
-    }
     public void setFlowToRemove(RemoveFlowInput flowToRemove) {
         this.flowToRemove = flowToRemove;
     }
     public Ipv4Address getGatewayIpAddress() {
         return gatewayIpAddress;
     }
-    public void setGatewayIpAddress(Ipv4Address gatewayIpAddress) {
-        this.gatewayIpAddress = gatewayIpAddress;
-    }
     public MacAddress getGatewayMacAddress() {
         return gatewayMacAddress;
     }
@@ -65,4 +56,72 @@ public final class ArpResolverMetadata {
         this.gatewayMacAddress = gatewayMacAddress;
     }
 
+    public Long getExternalNetworkBridgeDpid() {
+        return externalNetworkBridgeDpid;
+    }
+    public Ipv4Address getArpRequestSourceIp() {
+        return arpRequestSourceIp;
+    }
+    public MacAddress getArpRequestMacAddress() {
+        return arpRequestSourceMacAddress;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime
+                * result
+                + ((arpRequestSourceMacAddress == null) ? 0 : arpRequestSourceMacAddress
+                        .hashCode());
+        result = prime
+                * result
+                + ((arpRequestSourceIp == null) ? 0 : arpRequestSourceIp
+                        .hashCode());
+        result = prime
+                * result
+                + ((externalNetworkBridgeDpid == null) ? 0
+                        : externalNetworkBridgeDpid.hashCode());
+        result = prime
+                * result
+                + ((gatewayIpAddress == null) ? 0 : gatewayIpAddress.hashCode());
+        result = prime * result + (periodicRefresh ? 1231 : 1237);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        ArpResolverMetadata other = (ArpResolverMetadata) obj;
+        if (arpRequestSourceMacAddress == null) {
+            if (other.arpRequestSourceMacAddress != null)
+                return false;
+        } else if (!arpRequestSourceMacAddress.equals(other.arpRequestSourceMacAddress))
+            return false;
+        if (arpRequestSourceIp == null) {
+            if (other.arpRequestSourceIp != null)
+                return false;
+        } else if (!arpRequestSourceIp.equals(other.arpRequestSourceIp))
+            return false;
+        if (externalNetworkBridgeDpid == null) {
+            if (other.externalNetworkBridgeDpid != null)
+                return false;
+        } else if (!externalNetworkBridgeDpid
+                .equals(other.externalNetworkBridgeDpid))
+            return false;
+        if (gatewayIpAddress == null) {
+            if (other.gatewayIpAddress != null)
+                return false;
+        } else if (!gatewayIpAddress.equals(other.gatewayIpAddress))
+            return false;
+        if (periodicRefresh != other.periodicRefresh)
+            return false;
+        return true;
+    }
+
 }
index 3f404f684c66308d94b59ba974bd9bdf4b8f8443..7ccbb7de81c802844bf5e919230a5cdeb235445f 100644 (file)
@@ -13,8 +13,12 @@ import org.opendaylight.controller.liblldp.Ethernet;
 import org.opendaylight.controller.liblldp.NetUtils;
 import org.opendaylight.controller.liblldp.PacketException;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.packet.service.rev130709.PacketReceived;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ArpResolverUtils {
+    private static final Logger LOG = LoggerFactory.getLogger(ArpResolverUtils.class);
+
 
     static {
         Ethernet.etherTypeClassMap.put(EtherTypes.ARP.shortValue(), Arp.class);
@@ -26,15 +30,18 @@ public class ArpResolverUtils {
      *
      * @param potentialArp the packet for deserialization
      * @return ARP packet if received packet is ARP and deserialization was successful
-     * @throws PacketException if packet is not ARP or deserialization was not successful
      */
-    public static Arp getArpFrom(PacketReceived potentialArp) throws PacketException {
+    public static Arp getArpFrom(PacketReceived potentialArp) {
         byte[] payload = potentialArp.getPayload();
         Ethernet ethPkt = new Ethernet();
-        ethPkt.deserialize(payload, 0, payload.length * NetUtils.NumBitsInAByte);
+        try {
+            ethPkt.deserialize(payload, 0, payload.length * NetUtils.NumBitsInAByte);
+        } catch (PacketException e) {
+            LOG.trace("Failed to decode the incoming packet. ignoring it.");
+        }
         if (ethPkt.getPayload() instanceof Arp) {
             return (Arp) ethPkt.getPayload();
         }
-        throw new PacketException("Packet is not ARP: " + potentialArp);
+        return null;
     }
 }
index fad23d4b509499158ca3981665dfd57f06573d66..dd6ae714bc4411222cdfb9ce3dd9e327480f9574 100644 (file)
@@ -10,11 +10,14 @@ package org.opendaylight.ovsdb.openstack.netvirt.providers.openflow13.services.a
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.math.BigInteger;
-import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -96,12 +99,15 @@ public class GatewayMacResolverService extends AbstractServiceInstance
     private ArpSender arpSender;
     private SalFlowService flowService;
     private final AtomicLong flowCookie = new AtomicLong();
-    private final Map<Ipv4Address, ArpResolverMetadata> arpRemoveFlowInputAndL3EpKeyById =
+    private final ConcurrentMap<Ipv4Address, ArpResolverMetadata> gatewayToArpMetadataMap =
             new ConcurrentHashMap<Ipv4Address, ArpResolverMetadata>();
     private final int ARP_WATCH_BROTHERS = 10;
     private final int WAIT_CYCLES = 3;
     private final int PER_CYCLE_WAIT_DURATION = 1000;
+    private final int REFRESH_INTERVAL = 10;
     private final ListeningExecutorService arpWatcherWall = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(ARP_WATCH_BROTHERS));
+    private final ScheduledExecutorService gatewayMacRefresherPool = Executors.newScheduledThreadPool(1);
+    private final ScheduledExecutorService refreshRequester = Executors.newSingleThreadScheduledExecutor();
     private AtomicBoolean initializationDone = new AtomicBoolean(false);
 
     static {
@@ -134,6 +140,35 @@ public class GatewayMacResolverService extends AbstractServiceInstance
                 this.arpSender = null;
             }
             flowService = providerContext.getRpcService(SalFlowService.class);
+            refreshRequester.scheduleWithFixedDelay(new Runnable(){
+
+                @Override
+                public void run() {
+                    if (!gatewayToArpMetadataMap.isEmpty()){
+                        for(final Entry<Ipv4Address, ArpResolverMetadata> gatewayToArpMetadataEntry : gatewayToArpMetadataMap.entrySet()){
+                            final Ipv4Address gatewayIp = gatewayToArpMetadataEntry.getKey();
+                            final ArpResolverMetadata gatewayMetaData = gatewayToArpMetadataEntry.getValue();
+                            gatewayMacRefresherPool.schedule(new Runnable(){
+
+                                @Override
+                                public void run() {
+
+                                    final Node externalNetworkBridge = getExternalBridge(gatewayMetaData.getExternalNetworkBridgeDpid());
+                                    if(externalNetworkBridge == null){
+                                        LOG.error("MAC address for gateway {} can not be resolved, because external bridge {} "
+                                                + "is not connected to controller.",gatewayIp.getValue(),gatewayMetaData.getExternalNetworkBridgeDpid() );
+                                    }
+
+                                    LOG.debug("Refresh Gateway Mac for gateway {} using source ip {} and mac {} for ARP request",
+                                            gatewayIp.getValue(),gatewayMetaData.getArpRequestSourceIp().getValue(),gatewayMetaData.getArpRequestMacAddress().getValue());
+
+                                    sendGatewayArpRequest(externalNetworkBridge,gatewayIp,gatewayMetaData.getArpRequestSourceIp(), gatewayMetaData.getArpRequestMacAddress());
+                                }
+                            }, 1, TimeUnit.SECONDS);
+                        }
+                    }
+                }
+            }, REFRESH_INTERVAL, REFRESH_INTERVAL, TimeUnit.SECONDS);
         }
     }
     /**
@@ -162,35 +197,48 @@ public class GatewayMacResolverService extends AbstractServiceInstance
                 gatewayIp.getValue(),sourceIpAddress.getValue(),sourceMacAddress.getValue());
 
         init();
-        if(arpRemoveFlowInputAndL3EpKeyById.containsKey(gatewayIp)){
-            if(arpRemoveFlowInputAndL3EpKeyById.get(gatewayIp).getGatewayMacAddress() != null){
+        if(gatewayToArpMetadataMap.containsKey(gatewayIp)){
+            if(gatewayToArpMetadataMap.get(gatewayIp).getGatewayMacAddress() != null){
                 return arpWatcherWall.submit(new Callable<MacAddress>(){
 
                     @Override
                     public MacAddress call() throws Exception {
-                        return arpRemoveFlowInputAndL3EpKeyById.get(gatewayIp).getGatewayMacAddress();
+                        return gatewayToArpMetadataMap.get(gatewayIp).getGatewayMacAddress();
                     }
                 });
             }
         }else{
-            arpRemoveFlowInputAndL3EpKeyById.put(gatewayIp,
-                    new ArpResolverMetadata(null, gatewayIp,periodicRefresh));
+            gatewayToArpMetadataMap.put(gatewayIp,new ArpResolverMetadata(
+                    externalNetworkBridgeDpid, gatewayIp,sourceIpAddress,sourceMacAddress,periodicRefresh));
         }
 
-        final ArpMessageAddress senderAddress = new ArpMessageAddress(sourceMacAddress,sourceIpAddress);
-
-        final String nodeName = OPENFLOW + externalNetworkBridgeDpid;
 
-        final Node externalNetworkBridge = getOpenFlowNode(nodeName);
+        final Node externalNetworkBridge = getExternalBridge(externalNetworkBridgeDpid);
         if(externalNetworkBridge == null){
             LOG.error("MAC address for gateway {} can not be resolved, because external bridge {} "
                     + "is not connected to controller.",gatewayIp.getValue(),externalNetworkBridgeDpid );
             return null;
         }
+
+        sendGatewayArpRequest(externalNetworkBridge,gatewayIp,sourceIpAddress, sourceMacAddress);
+
+        //Wait for MacAddress population in cache
+        return waitForMacAddress(gatewayIp);
+    }
+
+    private Node getExternalBridge(final Long externalNetworkBridgeDpid){
+        final String nodeName = OPENFLOW + externalNetworkBridgeDpid;
+
+        return getOpenFlowNode(nodeName);
+    }
+
+    private void sendGatewayArpRequest(final Node externalNetworkBridge,final Ipv4Address gatewayIp,
+            final Ipv4Address sourceIpAddress, final MacAddress sourceMacAddress){
+        final ArpMessageAddress senderAddress = new ArpMessageAddress(sourceMacAddress,sourceIpAddress);
+
         //Build arp reply router flow
         final Flow arpReplyToControllerFlow = createArpReplyToControllerFlow(senderAddress, gatewayIp);
 
-
         final InstanceIdentifier<Node> nodeIid = InstanceIdentifier.builder(Nodes.class)
                 .child(Node.class, externalNetworkBridge.getKey())
                 .build();
@@ -213,7 +261,7 @@ public class GatewayMacResolverService extends AbstractServiceInstance
                 LOG.debug("Flow to route ARP Reply to Controller installed successfully : {}", flowIid);
 
                 //cache metadata
-                arpRemoveFlowInputAndL3EpKeyById.get(gatewayIp).setFlowToRemove(
+                gatewayToArpMetadataMap.get(gatewayIp).setFlowToRemove(
                         new RemoveFlowInputBuilder(arpReplyToControllerFlow).setNode(nodeRef).build());
 
                 //Broadcast ARP request packets
@@ -232,8 +280,6 @@ public class GatewayMacResolverService extends AbstractServiceInstance
             }
             }
         );
-        //Wait for MacAddress population in cache
-        return waitForMacAddress(gatewayIp);
     }
 
     private ListenableFuture<MacAddress> waitForMacAddress(final Ipv4Address gatewayIp){
@@ -246,10 +292,10 @@ public class GatewayMacResolverService extends AbstractServiceInstance
                     //Sleep before checking mac address, so meanwhile ARP request packets
                     // will be broadcasted on the bridge.
                     Thread.sleep(PER_CYCLE_WAIT_DURATION);
-                    ArpResolverMetadata arpResolverMetadata = arpRemoveFlowInputAndL3EpKeyById.get(gatewayIp);
+                    ArpResolverMetadata arpResolverMetadata = gatewayToArpMetadataMap.get(gatewayIp);
                     if(arpResolverMetadata != null && arpResolverMetadata.getGatewayMacAddress() != null){
                         if(!arpResolverMetadata.isPeriodicRefresh()){
-                            return arpRemoveFlowInputAndL3EpKeyById.remove(gatewayIp).getGatewayMacAddress();
+                            return gatewayToArpMetadataMap.remove(gatewayIp).getGatewayMacAddress();
                         }
                         return arpResolverMetadata.getGatewayMacAddress();
                     }
@@ -258,6 +304,7 @@ public class GatewayMacResolverService extends AbstractServiceInstance
             }
         });
     }
+
     private static @Nullable Ipv4Address getIPv4Addresses(IpAddress ipAddress) {
         if (ipAddress.getIpv4Address() == null) {
             return null;
@@ -320,33 +367,28 @@ public class GatewayMacResolverService extends AbstractServiceInstance
 
     @Override
     public void onPacketReceived(PacketReceived potentialArp) {
-        Arp arp = null;
-        try {
-            arp = ArpResolverUtils.getArpFrom(potentialArp);
-        } catch (Exception e) {
-            LOG.trace(
-                    "Failed to decode potential ARP packet. This could occur when other than ARP packet was received.",
-                    e);
-            return;
-        }
-        if (arp.getOperation() != ArpOperation.REPLY.intValue()) {
-            LOG.trace("Packet is not ARP REPLY packet.");
-            return;
-        }
-        if (LOG.isTraceEnabled()) {
-            LOG.trace("ARP REPLY received - {}", ArpUtils.getArpToStringFormat(arp));
-        }
-        NodeKey nodeKey = potentialArp.getIngress().getValue().firstKeyOf(Node.class, NodeKey.class);
-        if (nodeKey == null) {
-            LOG.info("Unknown source node of ARP packet: {}", potentialArp);
-            return;
-        }
-        Ipv4Address gatewayIpAddress = ArpUtils.bytesToIp(arp.getSenderProtocolAddress());
-        MacAddress gatewayMacAddress = ArpUtils.bytesToMac(arp.getSenderHardwareAddress());
-        ArpResolverMetadata removeFlowInputAndL3EpKey = arpRemoveFlowInputAndL3EpKeyById.get(gatewayIpAddress);
-        if(removeFlowInputAndL3EpKey != null){
-            removeFlowInputAndL3EpKey.setGatewayMacAddress(gatewayMacAddress);
-            flowService.removeFlow(removeFlowInputAndL3EpKey.getFlowToRemove());
+        Arp arp = ArpResolverUtils.getArpFrom(potentialArp);
+        if(arp != null){
+            if (arp.getOperation() != ArpOperation.REPLY.intValue()) {
+                LOG.trace("Packet is not ARP REPLY packet.");
+                return;
+            }
+            if (LOG.isTraceEnabled()) {
+                LOG.trace("ARP REPLY received - {}", ArpUtils.getArpToStringFormat(arp));
+            }
+            NodeKey nodeKey = potentialArp.getIngress().getValue().firstKeyOf(Node.class, NodeKey.class);
+            if (nodeKey == null) {
+                LOG.info("Unknown source node of ARP packet: {}", potentialArp);
+                return;
+            }
+            Ipv4Address gatewayIpAddress = ArpUtils.bytesToIp(arp.getSenderProtocolAddress());
+            MacAddress gatewayMacAddress = ArpUtils.bytesToMac(arp.getSenderHardwareAddress());
+            ArpResolverMetadata candidateGatewayIp = gatewayToArpMetadataMap.get(gatewayIpAddress);
+            if(candidateGatewayIp != null){
+                LOG.debug("Resolved MAC for Gateway Ip {} is {}",gatewayIpAddress.getValue(),gatewayMacAddress.getValue());
+                candidateGatewayIp.setGatewayMacAddress(gatewayMacAddress);
+                flowService.removeFlow(candidateGatewayIp.getFlowToRemove());
+            }
         }
     }
 
@@ -361,9 +403,9 @@ public class GatewayMacResolverService extends AbstractServiceInstance
     public void setDependencies(Object impl) {}
 
     @Override
-    public void stopPeriodicReferesh(Ipv4Address gatewayIp) {
+    public void stopPeriodicRefresh(Ipv4Address gatewayIp) {
         init();
-        arpRemoveFlowInputAndL3EpKeyById.remove(gatewayIp);
+        gatewayToArpMetadataMap.remove(gatewayIp);
     }
 
 }
index 28e6fc1964f1cebbbce9ff9a8b486c29b20abfac..28bf653b19e0c237d7a0ef118e767aabd7f363b3 100644 (file)
@@ -23,11 +23,15 @@ public interface GatewayMacResolver {
 
     /**
      * Method will trigger the mac resolution for gateway IP. If user set periodicRefresh to true,
-     * it will periodically trigger the gateway resolution after a specific time interval. If
-     * periodicRefresh is false, it will just do one time gateway resolution.
-     * @param externalNetworkBridgeDpid
-     * @param gatewayIp
-     * @param periodicReferesh
+     * it will periodically trigger the gateway resolution after a specific time interval using the
+     * given source IP and MAC addresses. It will also cache the source IP & MAC in case of periodicRefresh.
+     * If periodicRefresh is false, it will just do one time gateway resolution and won't cache any internal data.
+     * If user call the same method with different source ip and mac address, GatewayMacResolver service will
+     * update the internally cached data with these new source ip and mac address and will use it as per
+     * periodicRefresh flag.
+     * @param externalNetworkBridgeDpid This bridge will be used for sending ARP request
+     * @param gatewayIp ARP request will be send for this ip address
+     * @param periodicReferesh Do you want to periodically refresh the gateway mac?
      * @return
      */
     public ListenableFuture<MacAddress> resolveMacAddress( final Long externalNetworkBridgeDpid, final Ipv4Address gatewayIp,
@@ -37,5 +41,5 @@ public interface GatewayMacResolver {
      * Method will stop the periodic refresh of the given gateway ip address.
      * @param gatewayIp
      */
-    public void stopPeriodicReferesh(final Ipv4Address gatewayIp);
+    public void stopPeriodicRefresh(final Ipv4Address gatewayIp);
 }
index 651c79519dcade0abf22320b87e0380fe1ea033b..6d0801611c6e431f028b50a1a5467a502cbde3ee 100644 (file)
@@ -214,9 +214,9 @@ public class NeutronL3Adapter implements ConfigInterface {
 
                 if(externalNetwork != null){
                     if(externalNetwork.isRouterExternal()){
-                        final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
+                        final NeutronSubnet externalSubnet = neutronSubnetCache.getSubnet(neutronPort.getFixedIPs().get(0).getSubnetUUID());
                         if(externalSubnet != null){
-                            gatewayMacResolver.stopPeriodicReferesh(new Ipv4Address(externalSubnet.getGatewayIP()));
+                            gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
                         }
                     }
                 }
@@ -1280,14 +1280,16 @@ public class NeutronL3Adapter implements ConfigInterface {
                                         new Ipv4Address(externalSubnet.getGatewayIP()),
                                         new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
                                         new MacAddress(gatewayPort.getMacAddress()),
-                                        false);
+                                        true);
                         if(gatewayMacAddress != null){
                             Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
                                 @Override
                                 public void onSuccess(MacAddress result) {
                                     if(result != null){
-                                        updateExternalRouterMac(result.getValue());
-                                        LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(), result.getValue());
+                                        if(!result.getValue().equals(externalRouterMac)){
+                                            updateExternalRouterMac(result.getValue());
+                                            LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
+                                        }
                                     }else{
                                         LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
                                     }