ArpHandler to send ARP reply on proper vlan
[controller.git] / opendaylight / arphandler / src / main / java / org / opendaylight / controller / arphandler / internal / ArpHandler.java
index f3b22c75d8102a76beeb78229e7faabb89e60bd5..fe456f3f8ebe6f6237352cb09f5b6a7dc8056398 100644 (file)
@@ -1,4 +1,3 @@
-
 /*
  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
  *
@@ -37,22 +36,25 @@ import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
 import org.opendaylight.controller.clustering.services.IClusterServices;
 import org.opendaylight.controller.connectionmanager.IConnectionManager;
+import org.opendaylight.controller.hosttracker.HostIdFactory;
+import org.opendaylight.controller.hosttracker.IHostId;
 import org.opendaylight.controller.hosttracker.IfHostListener;
 import org.opendaylight.controller.hosttracker.IfIptoHost;
 import org.opendaylight.controller.hosttracker.hostAware.HostNodeConnector;
 import org.opendaylight.controller.hosttracker.hostAware.IHostFinder;
+import org.opendaylight.controller.sal.connection.ConnectionLocality;
 import org.opendaylight.controller.sal.core.ConstructionException;
 import org.opendaylight.controller.sal.core.Node;
 import org.opendaylight.controller.sal.core.NodeConnector;
 import org.opendaylight.controller.sal.packet.ARP;
 import org.opendaylight.controller.sal.packet.Ethernet;
 import org.opendaylight.controller.sal.packet.IDataPacketService;
+import org.opendaylight.controller.sal.packet.IEEE8021Q;
 import org.opendaylight.controller.sal.packet.IListenDataPacket;
 import org.opendaylight.controller.sal.packet.IPv4;
 import org.opendaylight.controller.sal.packet.Packet;
 import org.opendaylight.controller.sal.packet.PacketResult;
 import org.opendaylight.controller.sal.packet.RawPacket;
-import org.opendaylight.controller.sal.topology.TopoEdgeUpdate;
 import org.opendaylight.controller.sal.utils.EtherTypes;
 import org.opendaylight.controller.sal.utils.HexEncode;
 import org.opendaylight.controller.sal.utils.NetUtils;
@@ -62,6 +64,31 @@ import org.opendaylight.controller.topologymanager.ITopologyManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * The ArpHandler offers services to react on ARP requests and replies
+ * sent by network hosts. Moreover it allows for creating ARP messages
+ * by the controller itself.
+ *
+ * The ARP Handler on ODL doesn't use the requester MAC address in
+ * order to avoid to have to build a spanning tree where to forward
+ * ARP Requests. The ARP requests are broadcast packets so in order to
+ * reach everywhere need to be flooded, when you flood in a network
+ * that is not a tree (all the networks has some level of redundancy)
+ * that would create forwarding loops without a spanning tree. Given
+ * the need is only to send out the ARP requests toward all the hosts
+ * we actually don't need to implement a flooding mechanism in software
+ * (which would be expensive) we just send out the ARP request toward
+ * all the ports that are suspected to be host ports on all the
+ * switches (from the controller). Now the condition for which a port
+ * is marked as host port could potentially be incorrect so when the
+ * controller sends out the ARP Request that could come back to the
+ * controller and could cause another request not needed. So changing
+ * the source MAC address of the request to be the one of the controller,
+ * controller can protect itself from honoring twice the same request.
+ * This enables an ARP handler resolution, without the need of spanning
+ * tree and limiting software flooding to the minimum required.
+ */
+
 public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateAware<ARPEvent, Boolean> {
     private static final Logger log = LoggerFactory.getLogger(ArpHandler.class);
     static final String ARP_EVENT_CACHE_NAME = "arphandler.arpRequestReplyEvent";
@@ -76,28 +103,31 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
     private ConcurrentMap<InetAddress, Short> countDownTimers;
     private Timer periodicTimer;
     private BlockingQueue<ARPCacheEvent> ARPCacheEvents = new LinkedBlockingQueue<ARPCacheEvent>();
-    Thread cacheEventHandler;
+    private Thread cacheEventHandler;
+    private boolean stopping = false;
+
     /*
      * A cluster allocated cache. Used for synchronizing ARP request/reply
-     * events across all cluster controllers. To raise an event, we put() a specific
-     * event object (as key) and all nodes handle it in the entryUpdated callback.
+     * events across all cluster controllers. To raise an event, we put() a
+     * specific event object (as key) and all nodes handle it in the
+     * entryUpdated callback.
      *
      * In case of ARPReply, we put true value to send replies to any requestors
      * by calling generateAndSendReply
      */
     private ConcurrentMap<ARPEvent, Boolean> arpRequestReplyEvent;
 
-    void setConnectionManager(IConnectionManager cm){
+    void setConnectionManager(IConnectionManager cm) {
         this.connectionManager = cm;
     }
 
-    void unsetConnectionManager(IConnectionManager cm){
-        if (this.connectionManager == cm){
+    void unsetConnectionManager(IConnectionManager cm) {
+        if (this.connectionManager == cm) {
             connectionManager = null;
         }
     }
 
-    void setClusterContainerService(IClusterContainerServices s){
+    void setClusterContainerService(IClusterContainerServices s) {
         this.clusterContainerService = s;
     }
 
@@ -151,26 +181,18 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         }
     }
 
-    protected void sendARPReply(NodeConnector p, byte[] sMAC, InetAddress sIP,
-            byte[] tMAC, InetAddress tIP) {
+    protected void sendARPReply(NodeConnector p, byte[] sMAC, InetAddress sIP, byte[] tMAC, InetAddress tIP, short vlan) {
         byte[] senderIP = sIP.getAddress();
         byte[] targetIP = tIP.getAddress();
-        ARP arp = new ARP();
-        arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
-            .setProtocolType(EtherTypes.IPv4.shortValue())
-            .setHardwareAddressLength((byte) 6)
-            .setProtocolAddressLength((byte) 4)
-            .setOpCode(ARP.REPLY)
-            .setSenderHardwareAddress(sMAC)
-            .setSenderProtocolAddress(senderIP)
-            .setTargetHardwareAddress(tMAC)
-            .setTargetProtocolAddress(targetIP);
+        ARP arp = createARP(ARP.REPLY, sMAC, senderIP, tMAC, targetIP);
 
-        Ethernet ethernet = new Ethernet();
-        ethernet.setSourceMACAddress(sMAC)
-            .setDestinationMACAddress(tMAC)
-            .setEtherType(EtherTypes.ARP.shortValue())
-            .setPayload(arp);
+        if(log.isTraceEnabled()) {
+            log.trace("Sending Arp Reply with srcMac {} - srcIp {} - dstMac {} - dstIp {} - outport {}",
+                    HexEncode.bytesToHexString(sMAC),
+                    sIP, HexEncode.bytesToHexString(tMAC), tIP, p);
+        }
+
+        Ethernet ethernet = createEthernet(sMAC, tMAC, arp, vlan);
 
         RawPacket destPkt = this.dataPacketService.encodeDataPacket(ethernet);
         destPkt.setOutgoingNodeConnector(p);
@@ -178,7 +200,25 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         this.dataPacketService.transmitDataPacket(destPkt);
     }
 
-    protected void handleARPPacket(Ethernet eHeader, ARP pkt, NodeConnector p) {
+    private void logArpPacket(ARP pkt, NodeConnector p, short vlan) {
+        try {
+            log.trace("Received Arp {} with srcMac {} - srcIp {} - dstMac {} - dstIp {} - inport {} {}",
+                    ((pkt.getOpCode() == ARP.REQUEST) ? "Request" : "Reply"),
+                    HexEncode.bytesToHexString(pkt.getSenderHardwareAddress()),
+                    InetAddress.getByAddress(pkt.getSenderProtocolAddress()),
+                    HexEncode.bytesToHexString(pkt.getTargetHardwareAddress()),
+                    InetAddress.getByAddress(pkt.getTargetProtocolAddress()), p, (vlan != 0 ? "on vlan " + vlan : ""));
+
+        } catch (UnknownHostException e) {
+            log.warn("Illegal Ip Address in the ARP packet", e);
+        }
+    }
+
+    protected void handleARPPacket(Ethernet eHeader, ARP pkt, NodeConnector p, short vlan) {
+
+        if(log.isTraceEnabled()) {
+            logArpPacket(pkt, p, vlan);
+        }
 
         byte[] sourceMAC = eHeader.getSourceMACAddress();
         byte[] targetMAC = eHeader.getDestinationMACAddress();
@@ -214,17 +254,16 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
 
         // Make sure that the host is a legitimate member of this subnet
         if (!subnet.hasNodeConnector(p)) {
-            log.debug("{} showing up on {} does not belong to {}",
-                    new Object[] { sourceIP, p, subnet });
+            log.debug("{} showing up on {} does not belong to {}", new Object[] { sourceIP, p, subnet });
             return;
         }
 
         HostNodeConnector requestor = null;
         if (NetUtils.isUnicastMACAddr(sourceMAC) && p.getNode() != null) {
             try {
-                requestor = new HostNodeConnector(sourceMAC, sourceIP, p, subnet.getVlan());
+                requestor = new HostNodeConnector(sourceMAC, sourceIP, p, vlan);
             } catch (ConstructionException e) {
-                log.debug("Received ARP packet with invalid MAC: {}", sourceMAC);
+                log.debug("Received ARP packet with invalid MAC: {}", HexEncode.bytesToHexString(sourceMAC));
                 return;
             }
             /*
@@ -237,9 +276,9 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         }
 
         /*
-         * OpCode != request -> ARP Reply. If there are hosts (in
-         * arpRequestors) waiting for the ARP reply for this sourceIP, it's
-         * time to generate the reply and send it to these hosts.
+         * OpCode != request -> ARP Reply. If there are hosts (in arpRequestors)
+         * waiting for the ARP reply for this sourceIP, it's time to generate
+         * the reply and send it to these hosts.
          *
          * If sourceIP==targetIP, it is a Gratuitous ARP. If there are hosts (in
          * arpRequestors) waiting for the ARP reply for this sourceIP, it's time
@@ -247,40 +286,44 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
          */
 
         if (pkt.getOpCode() != ARP.REQUEST || sourceIP.equals(targetIP)) {
-            // Raise a reply event so that any waiting requestors will be sent a reply
-            // the true value indicates we should generate replies to requestors across the cluster
+            // Raise a reply event so that any waiting requestors will be sent a
+            // reply
+            // the true value indicates we should generate replies to requestors
+            // across the cluster
             log.trace("Received ARP reply packet from {}, reply to all requestors.", sourceIP);
-            arpRequestReplyEvent.put(new ARPReply(sourceIP, sourceMAC), true);
+            arpRequestReplyEvent.put(new ARPReply(sourceIP, sourceMAC, vlan), true);
             return;
         }
 
         /*
-         * ARP Request Handling:
-         * If targetIP is the IP of the subnet, reply with ARP REPLY
-         * If targetIP is a known host, PROXY ARP (by sending ARP REPLY) on behalf of known target hosts.
-         * For unknown target hosts, generate and send an ARP request to ALL switches/ports using
-         * the IP address defined in the subnet as source address
+         * ARP Request Handling: If targetIP is the IP of the subnet, reply with
+         * ARP REPLY If targetIP is a known host, PROXY ARP (by sending ARP
+         * REPLY) on behalf of known target hosts. For unknown target hosts,
+         * generate and send an ARP request to ALL switches/ports using the IP
+         * address defined in the subnet as source address
          */
         /*
          * If target IP is gateway IP, Send ARP reply
          */
         if ((targetIP.equals(subnet.getNetworkAddress()))
                 && (NetUtils.isBroadcastMACAddr(targetMAC) || Arrays.equals(targetMAC, getControllerMAC()))) {
-            if (connectionManager.isLocal(p.getNode())){
-                if (log.isTraceEnabled()){
-                    log.trace("Received local ARP req. for default gateway. Replying with controller MAC: {}", getControllerMAC());
+            if (connectionManager.getLocalityStatus(p.getNode()) == ConnectionLocality.LOCAL) {
+                if (log.isTraceEnabled()) {
+                    log.trace("Received local ARP req. for default gateway. Replying with controller MAC: {}",
+                            HexEncode.bytesToHexString(getControllerMAC()));
                 }
-                sendARPReply(p, getControllerMAC(), targetIP, pkt.getSenderHardwareAddress(), sourceIP);
+                sendARPReply(p, getControllerMAC(), targetIP, pkt.getSenderHardwareAddress(), sourceIP, vlan);
             } else {
                 log.trace("Received non-local ARP req. for default gateway. Raising reply event");
                 arpRequestReplyEvent.put(
-                        new ARPReply(p, targetIP, getControllerMAC(), sourceIP, pkt.getSenderHardwareAddress()), false);
+                        new ARPReply(p, targetIP, getControllerMAC(), sourceIP, pkt.getSenderHardwareAddress(), vlan), false);
             }
             return;
         }
 
-
-        HostNodeConnector host = hostTracker.hostQuery(targetIP);
+        // Hosttracker hosts db key implementation
+        IHostId id = HostIdFactory.create(targetIP, null);
+        HostNodeConnector host = hostTracker.hostQuery(id);
         // unknown host, initiate ARP request
         if (host == null) {
             // add the requestor to the list so that we can replay the reply
@@ -292,51 +335,43 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
                     arpRequestors.put(targetIP, requestorSet);
                 }
                 requestorSet.add(requestor);
-                countDownTimers.put(targetIP, (short) 2); // reset timeout to 2sec
+                countDownTimers.put(targetIP, (short) 2); // reset timeout to
+                                                          // 2sec
             }
-            //Raise a bcast request event, all controllers need to send one
+            // Raise a bcast request event, all controllers need to send one
             log.trace("Sending a bcast ARP request for {}", targetIP);
             arpRequestReplyEvent.put(new ARPRequest(targetIP, subnet), false);
-
         } else {
             /*
-             * Target host known (across the cluster), send ARP REPLY make sure that targetMAC
-             * matches the host's MAC if it is not broadcastMAC
+             * Target host known (across the cluster), send ARP REPLY make sure
+             * that targetMAC matches the host's MAC if it is not broadcastMAC
              */
             if (NetUtils.isBroadcastMACAddr(targetMAC) || Arrays.equals(host.getDataLayerAddressBytes(), targetMAC)) {
                 log.trace("Received ARP req. for known host {}, sending reply...", targetIP);
-                if (connectionManager.isLocal(p.getNode())) {
-                    sendARPReply(p,
-                            host.getDataLayerAddressBytes(),
-                            host.getNetworkAddress(),
-                            pkt.getSenderHardwareAddress(),
-                            sourceIP);
+                if (connectionManager.getLocalityStatus(p.getNode()) == ConnectionLocality.LOCAL) {
+                    sendARPReply(p, host.getDataLayerAddressBytes(), host.getNetworkAddress(),
+                            pkt.getSenderHardwareAddress(), sourceIP, vlan);
                 } else {
-                    arpRequestReplyEvent.put(new ARPReply(
-                            p,
-                            host.getNetworkAddress(),
-                            host.getDataLayerAddressBytes(),
-                            sourceIP,
-                            pkt.getSenderHardwareAddress()), false);
+                    arpRequestReplyEvent.put(new ARPReply(p, host.getNetworkAddress(), host.getDataLayerAddressBytes(),
+                            sourceIP, pkt.getSenderHardwareAddress(), vlan), false);
                 }
             } else {
                 /*
-                 * Target MAC has been changed. For now, discard it.
-                 * TODO: We may need to send unicast ARP REQUEST on behalf of
-                 * the target back to the sender to trigger the sender to update
-                 * its table
+                 * Target MAC has been changed. For now, discard it. TODO: We
+                 * may need to send unicast ARP REQUEST on behalf of the target
+                 * back to the sender to trigger the sender to update its table
                  */
             }
         }
     }
 
-    /*
-     *  Send a broadcast ARP Request to the switch/ ports  using
-     *  the networkAddress of the subnet as sender IP
-     *  the controller's MAC as sender MAC
-     *  the targetIP as the target Network Address
+    /**
+     * Send a broadcast ARP Request to the switch/ ports using the
+     * networkAddress of the subnet as sender IP the controller's MAC as sender
+     * MAC the targetIP as the target Network Address
      */
     protected void sendBcastARPRequest(InetAddress targetIP, Subnet subnet) {
+        log.trace("sendBcatARPRequest targetIP:{} subnet:{}", targetIP, subnet);
         Set<NodeConnector> nodeConnectors;
         if (subnet.isFlatLayer2()) {
             nodeConnectors = new HashSet<NodeConnector>();
@@ -346,36 +381,27 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         } else {
             nodeConnectors = subnet.getNodeConnectors();
         }
+        byte[] targetHardwareAddress = new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0 };
 
+        // TODO: should use IBroadcastHandler instead
         for (NodeConnector p : nodeConnectors) {
-
-            //fiter out any non-local or internal ports
-            if (! connectionManager.isLocal(p.getNode()) || topologyManager.isInternal(p)) {
+            // filter out any non-local or internal ports
+            if (!(connectionManager.getLocalityStatus(p.getNode()) == ConnectionLocality.LOCAL)
+                    || topologyManager.isInternal(p)) {
                 continue;
             }
-            ARP arp = new ARP();
+            log.trace("Sending toward nodeConnector:{}", p);
             byte[] senderIP = subnet.getNetworkAddress().getAddress();
             byte[] targetIPByte = targetIP.getAddress();
-            arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
-               .setProtocolType(EtherTypes.IPv4.shortValue())
-               .setHardwareAddressLength((byte) 6)
-               .setProtocolAddressLength((byte) 4)
-               .setOpCode(ARP.REQUEST)
-               .setSenderHardwareAddress(getControllerMAC())
-               .setSenderProtocolAddress(senderIP)
-               .setTargetHardwareAddress(
-                       new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0 })
-               .setTargetProtocolAddress(targetIPByte);
-
-            Ethernet ethernet = new Ethernet();
-            ethernet.setSourceMACAddress(getControllerMAC())
-                    .setDestinationMACAddress(new byte[] {(byte) -1,
-                                                          (byte) -1,
-                                                          (byte) -1,
-                                                          (byte) -1,
-                                                          (byte) -1,
-                                                          (byte) -1 })
-                    .setEtherType(EtherTypes.ARP.shortValue()).setPayload(arp);
+            ARP arp = createARP(ARP.REQUEST, getControllerMAC(), senderIP, targetHardwareAddress, targetIPByte);
+
+            if(log.isTraceEnabled()) {
+                log.trace("Sending Broadcast Arp Request with srcMac {} - srcIp {} - dstMac {} - dstIp {} - outport {}", HexEncode.bytesToHexString(getControllerMAC()),
+                        subnet.getNetworkAddress(), HexEncode.bytesToHexString(targetHardwareAddress), targetIP, p);
+            }
+
+            byte[] destMACAddress = NetUtils.getBroadcastMACAddr();
+            Ethernet ethernet = createEthernet(getControllerMAC(), destMACAddress, arp, (short)0);
 
             // TODO For now send port-by-port, see how to optimize to
             // send to multiple ports at once
@@ -386,14 +412,13 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         }
     }
 
-    /*
-     * Send a unicast ARP Request to the known host on a specific switch/port as
-     * defined in the host.
-     * The sender IP is the networkAddress of the subnet
-     * The sender MAC is the controller's MAC
+    /**
+     * Send a unicast ARP Request to the known host on specific (switch/port,
+     * vlan) as defined in the host. The sender IP is the networkAddress of the
+     * subnet The sender MAC is the controller's MAC
      */
     protected void sendUcastARPRequest(HostNodeConnector host, Subnet subnet) {
-
+        log.trace("sendUcastARPRequest host:{} subnet:{}", host, subnet);
         NodeConnector outPort = host.getnodeConnector();
         if (outPort == null) {
             log.error("Failed sending UcastARP because cannot extract output port from Host: {}", host);
@@ -403,22 +428,16 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         byte[] senderIP = subnet.getNetworkAddress().getAddress();
         byte[] targetIP = host.getNetworkAddress().getAddress();
         byte[] targetMAC = host.getDataLayerAddressBytes();
-        ARP arp = new ARP();
-        arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
-            .setProtocolType(EtherTypes.IPv4.shortValue())
-            .setHardwareAddressLength((byte) 6)
-            .setProtocolAddressLength((byte) 4)
-            .setOpCode(ARP.REQUEST)
-            .setSenderHardwareAddress(getControllerMAC())
-            .setSenderProtocolAddress(senderIP)
-            .setTargetHardwareAddress(targetMAC)
-            .setTargetProtocolAddress(targetIP);
+        ARP arp = createARP(ARP.REQUEST, getControllerMAC(), senderIP, targetMAC, targetIP);
 
-        Ethernet ethernet = new Ethernet();
-        ethernet.setSourceMACAddress(getControllerMAC())
-                .setDestinationMACAddress(targetMAC)
-                .setEtherType(EtherTypes.ARP.shortValue())
-                .setPayload(arp);
+        if(log.isTraceEnabled()) {
+            log.trace("Sending Unicast Arp Request with srcMac {} - srcIp {} - dstMac {} - dstIp {} - outport {}",
+                    HexEncode.bytesToHexString(getControllerMAC()),
+                    subnet.getNetworkAddress(), HexEncode.bytesToHexString(targetMAC), host.getNetworkAddress(),
+                    outPort);
+        }
+
+        Ethernet ethernet = createEthernet(getControllerMAC(), targetMAC, arp, host.getVlan());
 
         RawPacket destPkt = this.dataPacketService.encodeDataPacket(ethernet);
         destPkt.setOutgoingNodeConnector(outPort);
@@ -426,6 +445,7 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         this.dataPacketService.transmitDataPacket(destPkt);
     }
 
+    @Override
     public void find(InetAddress networkAddress) {
         log.trace("Received find IP {}", networkAddress);
 
@@ -445,20 +465,20 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
     /*
      * Probe the host by sending a unicast ARP Request to the host
      */
+    @Override
     public void probe(HostNodeConnector host) {
         log.trace("Received probe host {}", host);
 
         Subnet subnet = null;
         if (switchManager != null) {
-            subnet = switchManager.getSubnetByNetworkAddress(host
-                    .getNetworkAddress());
+            subnet = switchManager.getSubnetByNetworkAddress(host.getNetworkAddress());
         }
         if (subnet == null) {
             log.debug("can't find subnet matching {}", host.getNetworkAddress());
             return;
         }
 
-        if (connectionManager.isLocal(host.getnodeconnectorNode())){
+        if (connectionManager.getLocalityStatus(host.getnodeconnectorNode()) == ConnectionLocality.LOCAL) {
             log.trace("Send a ucast ARP req. to: {}", host);
             sendUcastARPRequest(host, subnet);
         } else {
@@ -467,18 +487,22 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         }
     }
 
-    /*
-     * An IP packet is punted to the controller, this means that the
-     * destination host is not known to the controller.
-     * Need to discover it by sending a Broadcast ARP Request
+    /**
+     * An IP packet is punted to the controller, this means that the destination
+     * host is not known to the controller. Need to discover it by sending a
+     * Broadcast ARP Request
+     *
+     * @param pkt
+     * @param p
      */
-    protected void handlePuntedIPPacket(IPv4 pkt, NodeConnector p) {
+    protected void handlePuntedIPPacket(IPv4 pkt, NodeConnector p, short vlan) {
 
         InetAddress dIP = NetUtils.getInetAddress(pkt.getDestinationAddress());
         if (dIP == null) {
-           return;
+            return;
         }
 
+        // try to find a matching subnet
         Subnet subnet = null;
         if (switchManager != null) {
             subnet = switchManager.getSubnetByNetworkAddress(dIP);
@@ -487,12 +511,27 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
             log.debug("Can't find subnet matching {}, drop packet", dIP);
             return;
         }
-        log.trace("Punted IP pkt from {}, sending bcast ARP event...", dIP);
-        /*
-         * unknown destination host, initiate bcast ARP request
-         */
-        arpRequestReplyEvent.put(new ARPRequest(dIP, subnet), false);
-        return;
+        // If packet is sent to the default gw (us), ignore it for now
+        if (subnet.getNetworkAddress().equals(dIP)) {
+            log.trace("Ignore IP packet destined to default gw");
+            return;
+        }
+
+        // see if we know about the host
+        // Hosttracker hosts db key implementation
+        HostNodeConnector host = hostTracker.hostFind(dIP);
+
+        if (host == null) {
+            // if we don't know about the host, try to find it
+            log.trace("Punted IP pkt to {}, sending bcast ARP event...", dIP);
+            /*
+             * unknown destination host, initiate bcast ARP request
+             */
+            arpRequestReplyEvent.put(new ARPRequest(dIP, subnet), false);
+
+        } else {
+            log.trace("Ignoring punted IP pkt to known host: {} (received on: {})", dIP, p);
+        }
     }
 
     public byte[] getControllerMAC() {
@@ -514,78 +553,80 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
 
         allocateCaches();
         retrieveCaches();
+
     }
 
-    @SuppressWarnings({ "unchecked", "deprecation" })
+    @SuppressWarnings({ "unchecked" })
     private void retrieveCaches() {
-        ConcurrentMap<?,?> map;
+        ConcurrentMap<?, ?> map;
 
-        if (this.clusterContainerService == null){
+        if (this.clusterContainerService == null) {
             log.error("Cluster service unavailable, can't retieve ARPHandler caches!");
             return;
         }
 
         map = clusterContainerService.getCache(ARP_EVENT_CACHE_NAME);
-        if (map != null){
+        if (map != null) {
             this.arpRequestReplyEvent = (ConcurrentMap<ARPEvent, Boolean>) map;
         } else {
             log.error("Cache allocation failed for {}", ARP_EVENT_CACHE_NAME);
         }
     }
 
-    @SuppressWarnings("deprecation")
     private void allocateCaches() {
-        if (clusterContainerService == null){
+        if (clusterContainerService == null) {
             nonClusterObjectCreate();
             log.error("Clustering service unavailable. Allocated non-cluster caches for ARPHandler.");
             return;
         }
 
-        try{
+        try {
             clusterContainerService.createCache(ARP_EVENT_CACHE_NAME,
                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
-        } catch (CacheConfigException e){
+        } catch (CacheConfigException e) {
             log.error("ARPHandler cache configuration invalid!");
-        } catch (CacheExistException e){
+        } catch (CacheExistException e) {
             log.debug("ARPHandler cache exists, skipped allocation.");
         }
 
     }
 
-    private void nonClusterObjectCreate(){
+    private void nonClusterObjectCreate() {
         arpRequestReplyEvent = new ConcurrentHashMap<ARPEvent, Boolean>();
     }
+
     /**
-     * Function called by the dependency manager when at least one
-     * dependency become unsatisfied or when the component is shutting
-     * down because for example bundle is being stopped.
+     * Function called by the dependency manager when at least one dependency
+     * become unsatisfied or when the component is shutting down because for
+     * example bundle is being stopped.
      *
      */
     void destroy() {
+        cacheEventHandler.interrupt();
     }
 
     /**
-     * Function called by dependency manager after "init ()" is called
-     * and after the services provided by the class are registered in
-     * the service registry
+     * Function called by dependency manager after "init ()" is called and after
+     * the services provided by the class are registered in the service registry
      *
      */
     void start() {
+        stopping = false;
         startPeriodicTimer();
         cacheEventHandler.start();
-
     }
 
     /**
-     * Function called by the dependency manager before the services
-     * exported by the component are unregistered, this will be
-     * followed by a "destroy ()" calls
+     * Function called by the dependency manager before the services exported by
+     * the component are unregistered, this will be followed by a "destroy ()"
+     * calls
      *
      */
-    void stop(){
+    void stop() {
     }
 
     void stopping() {
+        stopping = true;
         cancelPeriodicTimer();
     }
 
@@ -609,23 +650,60 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         log.trace("Received a frame of size: {}", inPkt.getPacketData().length);
         Packet formattedPak = this.dataPacketService.decodeDataPacket(inPkt);
         if (formattedPak instanceof Ethernet) {
-            Object nextPak = formattedPak.getPayload();
+            Packet nextPak = formattedPak.getPayload();
+            short vlan = 0;
+            if (nextPak instanceof IEEE8021Q) {
+                vlan = ((IEEE8021Q) nextPak).getVid();
+                log.trace("Moved after the dot1Q header");
+                nextPak = ((IEEE8021Q) nextPak).getPayload();
+            }
             if (nextPak instanceof IPv4) {
                 log.trace("Handle IP packet: {}", formattedPak);
-                handlePuntedIPPacket((IPv4) nextPak, inPkt.getIncomingNodeConnector());
+                handlePuntedIPPacket((IPv4) nextPak, inPkt.getIncomingNodeConnector(), vlan);
             } else if (nextPak instanceof ARP) {
                 log.trace("Handle ARP packet: {}", formattedPak);
-                handleARPPacket((Ethernet) formattedPak, (ARP) nextPak, inPkt
-                        .getIncomingNodeConnector());
+                handleARPPacket((Ethernet) formattedPak, (ARP) nextPak, inPkt.getIncomingNodeConnector(), vlan);
             }
         }
         return PacketResult.IGNORED;
     }
 
+    private ARP createARP(short opCode, byte[] senderMacAddress, byte[] senderIP, byte[] targetMacAddress,
+            byte[] targetIP) {
+        ARP arp = new ARP();
+        arp.setHardwareType(ARP.HW_TYPE_ETHERNET);
+        arp.setProtocolType(EtherTypes.IPv4.shortValue());
+        arp.setHardwareAddressLength((byte) 6);
+        arp.setProtocolAddressLength((byte) 4);
+        arp.setOpCode(opCode);
+        arp.setSenderHardwareAddress(senderMacAddress);
+        arp.setSenderProtocolAddress(senderIP);
+        arp.setTargetHardwareAddress(targetMacAddress);
+        arp.setTargetProtocolAddress(targetIP);
+        return arp;
+    }
+
+    private Ethernet createEthernet(byte[] sourceMAC, byte[] targetMAC, ARP arp, short vlan) {
+        Ethernet ethernet = new Ethernet();
+        ethernet.setSourceMACAddress(sourceMAC);
+        ethernet.setDestinationMACAddress(targetMAC);
+        if (vlan == 0) {
+            ethernet.setEtherType(EtherTypes.ARP.shortValue());
+            ethernet.setPayload(arp);
+        } else {
+            IEEE8021Q dot1q = new IEEE8021Q();
+            dot1q.setVid(vlan);
+            dot1q.setEtherType(EtherTypes.ARP.shortValue());
+            dot1q.setPayload(arp);
+            ethernet.setEtherType(EtherTypes.VLANTAGGED.shortValue());
+            ethernet.setPayload(dot1q);
+        }
+        return ethernet;
+    }
+
     private void startPeriodicTimer() {
         this.periodicTimer = new Timer("ArpHandler Periodic Timer");
         this.periodicTimer.scheduleAtFixedRate(new TimerTask() {
-            @SuppressWarnings("deprecation")
             @Override
             public void run() {
                 Set<InetAddress> targetIPs = countDownTimers.keySet();
@@ -649,10 +727,10 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
 
                 // Clean up ARP event cache
                 try {
-                    if (clusterContainerService.amICoordinator() && ! arpRequestReplyEvent.isEmpty()){
+                    if (clusterContainerService.amICoordinator() && !arpRequestReplyEvent.isEmpty()) {
                         arpRequestReplyEvent.clear();
                     }
-                } catch (Exception e){
+                } catch (Exception e) {
                     log.warn("ARPHandler: A cluster member failed to clear event cache.");
                 }
             }
@@ -665,37 +743,44 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
         }
     }
 
-    private void generateAndSendReply(InetAddress sourceIP, byte[] sourceMAC) {
+    private void generateAndSendReply(InetAddress sourceIP, byte[] sourceMAC, short vlan) {
+        if (log.isTraceEnabled()) {
+            log.trace("generateAndSendReply called with params sourceIP:{} sourceMAC:{}", sourceIP,
+                    HexEncode.bytesToHexString(sourceMAC));
+        }
         Set<HostNodeConnector> hosts = arpRequestors.remove(sourceIP);
         if ((hosts == null) || hosts.isEmpty()) {
+            log.trace("Bailing out no requestors Hosts");
             return;
         }
         countDownTimers.remove(sourceIP);
         for (HostNodeConnector host : hosts) {
-            log.trace("Sending ARP Reply with src {}/{}, target {}/{}",
-                    new Object[] { sourceMAC, sourceIP, host.getDataLayerAddressBytes(), host.getNetworkAddress() });
-
-            if (connectionManager.isLocal(host.getnodeconnectorNode())){
-                sendARPReply(host.getnodeConnector(),
-                        sourceMAC,
-                        sourceIP,
-                        host.getDataLayerAddressBytes(),
-                        host.getNetworkAddress());
+            if (log.isTraceEnabled()) {
+                log.trace(
+                        "Sending ARP Reply with src {}/{}, target {}/{} {}",
+                        new Object[] { HexEncode.bytesToHexString(sourceMAC), sourceIP,
+                                HexEncode.bytesToHexString(host.getDataLayerAddressBytes()), host.getNetworkAddress(),
+                                (vlan != 0 ? "on vlan " + vlan : "") });
+            }
+            if (connectionManager.getLocalityStatus(host.getnodeconnectorNode()) == ConnectionLocality.LOCAL) {
+                sendARPReply(host.getnodeConnector(), sourceMAC, sourceIP, host.getDataLayerAddressBytes(),
+                        host.getNetworkAddress(), vlan);
             } else {
+                /*
+                 * In the remote event a requestor moved to another controller
+                 * it may turn out it now we need to send the ARP reply from a
+                 * different controller, this cover the case
+                 */
                 arpRequestReplyEvent.put(
-                        new ARPReply(
-                            host.getnodeConnector(),
-                            sourceIP,
-                            sourceMAC,
-                            host.getNetworkAddress(),
-                            host.getDataLayerAddressBytes()), false);
+                        new ARPReply(host.getnodeConnector(), sourceIP, sourceMAC, host.getNetworkAddress(), host
+                                .getDataLayerAddressBytes(), vlan), false);
             }
         }
     }
 
-
     @Override
     public void entryUpdated(ARPEvent key, Boolean new_value, String cacheName, boolean originLocal) {
+        log.trace("Got and entryUpdated for cacheName {} key {} isNew {}", cacheName, key, new_value);
         enqueueARPCacheEvent(key, new_value);
     }
 
@@ -703,16 +788,18 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
     public void entryCreated(ARPEvent key, String cacheName, boolean originLocal) {
         // nothing to do
     }
+
     @Override
     public void entryDeleted(ARPEvent key, String cacheName, boolean originLocal) {
         // nothing to do
     }
 
-    private void enqueueARPCacheEvent (ARPEvent event, boolean new_value) {
+    private void enqueueARPCacheEvent(ARPEvent event, boolean new_value) {
         try {
             ARPCacheEvent cacheEvent = new ARPCacheEvent(event, new_value);
             if (!ARPCacheEvents.contains(cacheEvent)) {
                 this.ARPCacheEvents.add(cacheEvent);
+                log.trace("Enqueued {}", event);
             }
         } catch (Exception e) {
             log.debug("enqueueARPCacheEvent caught Interrupt Exception for event {}", event);
@@ -720,12 +807,13 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
     }
 
     /*
-     * this thread monitors the connectionEvent queue for new incoming events from
+     * this thread monitors the connectionEvent queue for new incoming events
+     * from
      */
     private class ARPCacheEventHandler implements Runnable {
         @Override
         public void run() {
-            while (true) {
+            while (!stopping) {
                 try {
                     ARPCacheEvent ev = ARPCacheEvents.take();
                     ARPEvent event = ev.getEvent();
@@ -733,25 +821,26 @@ public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateA
                         ARPRequest req = (ARPRequest) event;
                         // If broadcast request
                         if (req.getHost() == null) {
+                            log.trace("Trigger and ARP Broadcast Request upon receipt of {}", req);
                             sendBcastARPRequest(req.getTargetIP(), req.getSubnet());
 
-                        //If unicast and local, send reply
-                        } else if (connectionManager.isLocal(req.getHost().getnodeconnectorNode())) {
+                            // If unicast and local, send reply
+                        } else if (connectionManager.getLocalityStatus(req.getHost().getnodeconnectorNode()) == ConnectionLocality.LOCAL) {
+                            log.trace("ARPCacheEventHandler - sendUcatARPRequest upon receipt of {}", req);
                             sendUcastARPRequest(req.getHost(), req.getSubnet());
                         }
                     } else if (event instanceof ARPReply) {
                         ARPReply rep = (ARPReply) event;
-                        // New reply received by controller, notify all awaiting requestors across the cluster
+                        // New reply received by controller, notify all awaiting
+                        // requestors across the cluster
                         if (ev.isNewReply()) {
-                            generateAndSendReply(rep.getTargetIP(), rep.getTargetMac());
-
-                        // Otherwise, a specific reply. If local, send out.
-                        } else if (connectionManager.isLocal(rep.getPort().getNode())) {
-                            sendARPReply(rep.getPort(),
-                                    rep.getSourceMac(),
-                                    rep.getSourceIP(),
-                                    rep.getTargetMac(),
-                                    rep.getTargetIP());
+                            log.trace("Trigger a generateAndSendReply in response to {}", rep);
+                            generateAndSendReply(rep.getTargetIP(), rep.getTargetMac(), rep.getVlan());
+                            // Otherwise, a specific reply. If local, send out.
+                        } else if (connectionManager.getLocalityStatus(rep.getPort().getNode()) == ConnectionLocality.LOCAL) {
+                            log.trace("ARPCacheEventHandler - sendUcatARPReply locally in response to {}", rep);
+                            sendARPReply(rep.getPort(), rep.getSourceMac(), rep.getSourceIP(), rep.getTargetMac(),
+                                    rep.getTargetIP(), rep.getVlan());
                         }
                     }
                 } catch (InterruptedException e) {