HA - ARPHandler Event sync
[controller.git] / opendaylight / arphandler / src / main / java / org / opendaylight / controller / arphandler / internal / ArpHandler.java
index 811c7aca8370525d13bcabd2ef46d898bcc4a88f..0ff1cd9bd6a542dc8694b09933bfe2c1835d3bbd 100644 (file)
@@ -16,13 +16,24 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-
+import java.util.concurrent.CopyOnWriteArraySet;
+
+import org.opendaylight.controller.arphandler.ARPEvent;
+import org.opendaylight.controller.arphandler.ARPReply;
+import org.opendaylight.controller.arphandler.ARPRequest;
+import org.opendaylight.controller.clustering.services.CacheConfigException;
+import org.opendaylight.controller.clustering.services.CacheExistException;
+import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
+import org.opendaylight.controller.clustering.services.IClusterContainerServices;
+import org.opendaylight.controller.clustering.services.IClusterServices;
+import org.opendaylight.controller.connectionmanager.IConnectionManager;
 import org.opendaylight.controller.hosttracker.IfHostListener;
 import org.opendaylight.controller.hosttracker.IfIptoHost;
 import org.opendaylight.controller.hosttracker.hostAware.HostNodeConnector;
@@ -31,7 +42,6 @@ import org.opendaylight.controller.sal.core.ConstructionException;
 import org.opendaylight.controller.sal.core.Node;
 import org.opendaylight.controller.sal.core.NodeConnector;
 import org.opendaylight.controller.sal.packet.ARP;
-import org.opendaylight.controller.sal.packet.BitBufferHelper;
 import org.opendaylight.controller.sal.packet.Ethernet;
 import org.opendaylight.controller.sal.packet.IDataPacketService;
 import org.opendaylight.controller.sal.packet.IListenDataPacket;
@@ -48,28 +58,58 @@ import org.opendaylight.controller.topologymanager.ITopologyManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class ArpHandler implements IHostFinder, IListenDataPacket {
-    private static final Logger logger = LoggerFactory
-            .getLogger(ArpHandler.class);
-    private IfIptoHost hostTracker = null;
-    private ISwitchManager switchManager = null;
+public class ArpHandler implements IHostFinder, IListenDataPacket, ICacheUpdateAware<ARPEvent, Boolean> {
+    private static final Logger log = LoggerFactory.getLogger(ArpHandler.class);
+    static final String ARP_EVENT_CACHE_NAME = "arphandler.arpRequestReplyEvent";
+    private IfIptoHost hostTracker;
+    private ISwitchManager switchManager;
     private ITopologyManager topologyManager;
-    private IDataPacketService dataPacketService = null;
-    private Set<IfHostListener> hostListener = Collections
-            .synchronizedSet(new HashSet<IfHostListener>());
+    private IDataPacketService dataPacketService;
+    private IClusterContainerServices clusterContainerService;
+    private IConnectionManager connectionManager;
+    private Set<IfHostListener> hostListeners = new CopyOnWriteArraySet<IfHostListener>();
     private ConcurrentMap<InetAddress, Set<HostNodeConnector>> arpRequestors;
     private ConcurrentMap<InetAddress, Short> countDownTimers;
     private Timer periodicTimer;
+    /*
+     * A cluster allocated cache. Used for synchronizing ARP request/reply
+     * events across all cluster controllers. To raise an event, we put() a specific
+     * event object (as key) and all nodes handle it in the entryUpdated callback.
+     *
+     * In case of ARPReply, we put true value to send replies to any requestors
+     * by calling generateAndSendReply
+     */
+    private ConcurrentMap<ARPEvent, Boolean> arpRequestReplyEvent;
+
+    void setConnectionManager(IConnectionManager cm){
+        this.connectionManager = cm;
+    }
+
+    void unsetConnectionManager(IConnectionManager cm){
+        if (this.connectionManager == cm){
+            connectionManager = null;
+        }
+    }
+
+    void setClusterContainerService(IClusterContainerServices s){
+        this.clusterContainerService = s;
+    }
+
+    void unsetClusterContainerService(IClusterContainerServices s) {
+        if (this.clusterContainerService == s) {
+            this.clusterContainerService = null;
+        }
+    }
 
     void setHostListener(IfHostListener s) {
-        if (this.hostListener != null) {
-            this.hostListener.add(s);
+        if (this.hostListeners != null) {
+            this.hostListeners.add(s);
         }
     }
 
     void unsetHostListener(IfHostListener s) {
-        if (this.hostListener != null) {
-            this.hostListener.remove(s);
+        if (this.hostListeners != null) {
+            this.hostListeners.remove(s);
         }
     }
 
@@ -83,17 +123,13 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         }
     }
 
-    public IfIptoHost getHostTracker() {
-        return hostTracker;
-    }
-
     public void setHostTracker(IfIptoHost hostTracker) {
-        logger.debug("Setting HostTracker");
+        log.debug("Setting HostTracker");
         this.hostTracker = hostTracker;
     }
 
     public void unsetHostTracker(IfIptoHost s) {
-        logger.debug("UNSetting HostTracker");
+        log.debug("UNSetting HostTracker");
         if (this.hostTracker == s) {
             this.hostTracker = null;
         }
@@ -114,17 +150,21 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         byte[] senderIP = sIP.getAddress();
         byte[] targetIP = tIP.getAddress();
         ARP arp = new ARP();
-        arp.setHardwareType(ARP.HW_TYPE_ETHERNET).setProtocolType(
-                EtherTypes.IPv4.shortValue())
-                .setHardwareAddressLength((byte) 6).setProtocolAddressLength(
-                        (byte) 4).setOpCode(ARP.REPLY)
-                .setSenderHardwareAddress(sMAC).setSenderProtocolAddress(
-                        senderIP).setTargetHardwareAddress(tMAC)
-                .setTargetProtocolAddress(targetIP);
+        arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
+            .setProtocolType(EtherTypes.IPv4.shortValue())
+            .setHardwareAddressLength((byte) 6)
+            .setProtocolAddressLength((byte) 4)
+            .setOpCode(ARP.REPLY)
+            .setSenderHardwareAddress(sMAC)
+            .setSenderProtocolAddress(senderIP)
+            .setTargetHardwareAddress(tMAC)
+            .setTargetProtocolAddress(targetIP);
 
         Ethernet ethernet = new Ethernet();
-        ethernet.setSourceMACAddress(sMAC).setDestinationMACAddress(tMAC)
-                .setEtherType(EtherTypes.ARP.shortValue()).setPayload(arp);
+        ethernet.setSourceMACAddress(sMAC)
+            .setDestinationMACAddress(tMAC)
+            .setEtherType(EtherTypes.ARP.shortValue())
+            .setPayload(arp);
 
         RawPacket destPkt = this.dataPacketService.encodeDataPacket(ethernet);
         destPkt.setOutgoingNodeConnector(p);
@@ -132,114 +172,79 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         this.dataPacketService.transmitDataPacket(destPkt);
     }
 
-    private boolean isBroadcastMAC(byte[] mac) {
-        if (BitBufferHelper.toNumber(mac) == 0xffffffffffffL) { //TODO: implement this in our Ethernet
-            return true;
-        }
-        return false;
-    }
-
-    private boolean isUnicastMAC(byte[] mac) {
-        if ((BitBufferHelper.toNumber(mac) & 0x010000000000L) == 0) {
-            return true;
-        }
-        return false;
-    }
-
     protected void handleARPPacket(Ethernet eHeader, ARP pkt, NodeConnector p) {
-        if (pkt.getOpCode() == 0x1) {
-            logger.debug("Received ARP REQUEST Packet from NodeConnector: {}",
-                         p);
-        } else {
-            logger.debug("Received ARP REPLY Packet from NodeConnector: {}",
-                         p);
-        }
-        InetAddress targetIP = null;
-        try {
-            targetIP = InetAddress.getByAddress(pkt.getTargetProtocolAddress());
-        } catch (UnknownHostException e1) {
-            return;
-        }
-        InetAddress sourceIP = null;
-        try {
-            sourceIP = InetAddress.getByAddress(pkt.getSenderProtocolAddress());
-        } catch (UnknownHostException e1) {
-            return;
-        }
-        byte[] targetMAC = eHeader.getDestinationMACAddress();
-        byte[] sourceMAC = eHeader.getSourceMACAddress();
 
+        byte[] sourceMAC = eHeader.getSourceMACAddress();
+        byte[] targetMAC = eHeader.getDestinationMACAddress();
         /*
          * Sanity Check; drop ARP packets originated by the controller itself.
          * This is to avoid continuous flooding
          */
         if (Arrays.equals(sourceMAC, getControllerMAC())) {
-            if (logger.isDebugEnabled()) {
-              logger.debug(
-                    "Receive the self originated packet (srcMAC {}) --> DROP",
-                    HexEncode.bytesToHexString(sourceMAC));
+            if (log.isDebugEnabled()) {
+                log.debug("Receive a self originated ARP pkt (srcMAC {}) --> DROP",
+                        HexEncode.bytesToHexString(sourceMAC));
             }
             return;
         }
 
+        InetAddress targetIP, sourceIP;
+        try {
+            targetIP = InetAddress.getByAddress(pkt.getTargetProtocolAddress());
+            sourceIP = InetAddress.getByAddress(pkt.getSenderProtocolAddress());
+        } catch (UnknownHostException e1) {
+            log.debug("Invalid host in ARP packet: {}", e1.getMessage());
+            return;
+        }
+
         Subnet subnet = null;
         if (switchManager != null) {
             subnet = switchManager.getSubnetByNetworkAddress(sourceIP);
         }
         if (subnet == null) {
-            logger.debug("can't find subnet matching {}, drop packet",sourceIP);
+            log.debug("ARPHandler: can't find subnet matching {}, drop packet", sourceIP);
             return;
         }
-        logger.debug("Found {} matching {}", subnet, sourceIP);
-        /*
-         * Make sure that the host is a legitimate member of this subnet
-         */
+
+        // Make sure that the host is a legitimate member of this subnet
         if (!subnet.hasNodeConnector(p)) {
-            logger.debug("{} showing up on {} does not belong to {}",
+            log.debug("{} showing up on {} does not belong to {}",
                     new Object[] { sourceIP, p, subnet });
             return;
         }
 
         HostNodeConnector requestor = null;
-        if (isUnicastMAC(sourceMAC)) {
-            // TODO For not this is only OPENFLOW but we need to fix this
-            if (p.getType().equals(
-                    NodeConnector.NodeConnectorIDType.OPENFLOW)) {
-                try {
-                    requestor = new HostNodeConnector(sourceMAC, sourceIP, p, subnet
-                            .getVlan());
-                } catch (ConstructionException e) {
-                    return;
-                }
-                /*
-                 * Learn host from the received ARP REQ/REPLY, inform
-                 * Host Tracker
-                 */
-                logger.debug("Inform Host tracker of new host {}", requestor.getNetworkAddress());
-                synchronized (this.hostListener) {
-                    for (IfHostListener listener : this.hostListener) {
-                        listener.hostListener(requestor);
-                    }
-                }
+        if (NetUtils.isUnicastMACAddr(sourceMAC) && p.getNode() != null) {
+            try {
+                requestor = new HostNodeConnector(sourceMAC, sourceIP, p, subnet.getVlan());
+            } catch (ConstructionException e) {
+                log.debug("Received ARP packet with invalid MAC: {}", sourceMAC);
+                return;
+            }
+            /*
+             * Learn host from the received ARP REQ/REPLY, inform Host Tracker
+             */
+            log.trace("Inform Host tracker of new host {}", requestor.getNetworkAddress());
+            for (IfHostListener listener : this.hostListeners) {
+                listener.hostListener(requestor);
             }
-        }
-        /*
-         * Gratuitous ARP. If there are hosts (in arpRequestors) waiting for the
-         * ARP reply for this sourceIP, it's time to generate the reply and it
-         * to these hosts
-         */
-        if (sourceIP.equals(targetIP)) {
-            generateAndSendReply(sourceIP, sourceMAC);
-            return;
         }
 
         /*
-         * ARP Reply. If there are hosts (in arpRequesttors) waiting for the ARP
-         * reply for this sourceIP, it's time to generate the reply and it to
-         * these hosts
+         * OpCode != request -> ARP Reply. If there are hosts (in
+         * arpRequestors) waiting for the ARP reply for this sourceIP, it's
+         * time to generate the reply and send it to these hosts.
+         *
+         * If sourceIP==targetIP, it is a Gratuitous ARP. If there are hosts (in
+         * arpRequestors) waiting for the ARP reply for this sourceIP, it's time
+         * to generate the reply and send it to these hosts
          */
-        if (pkt.getOpCode() != ARP.REQUEST) {
-            generateAndSendReply(sourceIP, sourceMAC);
+
+        if (pkt.getOpCode() != ARP.REQUEST || sourceIP.equals(targetIP)) {
+            // Raise a reply event so that any waiting requestors will be sent a reply
+            // the true value indicates we should generate replies to requestors across the cluster
+            log.trace("Received ARP reply packet from {}, reply to all requestors.", sourceIP);
+            arpRequestReplyEvent.put(new ARPReply(sourceIP, sourceMAC), true);
             return;
         }
 
@@ -251,55 +256,71 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
          * the IP address defined in the subnet as source address
          */
         /*
-         * Send ARP reply if target IP is gateway IP
+         * If target IP is gateway IP, Send ARP reply
          */
         if ((targetIP.equals(subnet.getNetworkAddress()))
-                && (isBroadcastMAC(targetMAC) || Arrays.equals(targetMAC,
-                        getControllerMAC()))) {
-            sendARPReply(p, getControllerMAC(), targetIP, pkt
-                    .getSenderHardwareAddress(), sourceIP);
+                && (NetUtils.isBroadcastMACAddr(targetMAC) || Arrays.equals(targetMAC, getControllerMAC()))) {
+            if (connectionManager.isLocal(p.getNode())){
+                if (log.isTraceEnabled()){
+                    log.trace("Received local ARP req. for default gateway. Replying with controller MAC: {}", getControllerMAC());
+                }
+                sendARPReply(p, getControllerMAC(), targetIP, pkt.getSenderHardwareAddress(), sourceIP);
+            } else {
+                log.trace("Received non-local ARP req. for default gateway. Raising reply event");
+                arpRequestReplyEvent.put(
+                        new ARPReply(p, targetIP, getControllerMAC(), sourceIP, pkt.getSenderHardwareAddress()), false);
+            }
             return;
         }
 
-        /*
-         * unknown host, initiate ARP request
-         */
+
         HostNodeConnector host = hostTracker.hostQuery(targetIP);
+        // unknown host, initiate ARP request
         if (host == null) {
             // add the requestor to the list so that we can replay the reply
             // when the host responds
             if (requestor != null) {
-                Set<HostNodeConnector> requestorSet = arpRequestors
-                        .get(targetIP);
-                if ((requestorSet == null) || requestorSet.isEmpty()) {
-                    requestorSet = new HashSet<HostNodeConnector>();
-                    countDownTimers.put(targetIP, (short) 2); // set max timeout
-                                                              // to 2sec
+                Set<HostNodeConnector> requestorSet = arpRequestors.get(targetIP);
+                if (requestorSet == null) {
+                    requestorSet = Collections.newSetFromMap(new ConcurrentHashMap<HostNodeConnector, Boolean>());
+                    arpRequestors.put(targetIP, requestorSet);
                 }
                 requestorSet.add(requestor);
-                arpRequestors.put(targetIP, requestorSet);
+                countDownTimers.put(targetIP, (short) 2); // reset timeout to 2sec
             }
-            sendBcastARPRequest(targetIP, subnet);
-            return;
-        }
-        /*
-         * Known target host, send ARP REPLY
-         * make sure that targetMAC matches the host's MAC if it is not broadcastMAC
-         */
-        if (isBroadcastMAC(targetMAC)
-                || Arrays.equals(host.getDataLayerAddressBytes(), targetMAC)) {
-            sendARPReply(p, host.getDataLayerAddressBytes(), host
-                    .getNetworkAddress(), pkt.getSenderHardwareAddress(),
-                    sourceIP);
-            return;
+            //Raise a bcast request event, all controllers need to send one
+            log.trace("Sending a bcast ARP request for {}", targetIP);
+            arpRequestReplyEvent.put(new ARPRequest(targetIP, subnet), false);
+
         } else {
             /*
-             * target target MAC has been changed. For now, discard it.
-             * TODO: We may need to send unicast ARP REQUEST on behalf of the
-             * target back to the sender to trigger the sender to
-             * update its table
+             * Target host known (across the cluster), send ARP REPLY make sure that targetMAC
+             * matches the host's MAC if it is not broadcastMAC
              */
-            return;
+            if (NetUtils.isBroadcastMACAddr(targetMAC) || Arrays.equals(host.getDataLayerAddressBytes(), targetMAC)) {
+                log.trace("Received ARP req. for known host {}, sending reply...", targetIP);
+                if (connectionManager.isLocal(p.getNode())) {
+                    sendARPReply(p,
+                            host.getDataLayerAddressBytes(),
+                            host.getNetworkAddress(),
+                            pkt.getSenderHardwareAddress(),
+                            sourceIP);
+                } else {
+                    arpRequestReplyEvent.put(new ARPReply(
+                            p,
+                            host.getNetworkAddress(),
+                            host.getDataLayerAddressBytes(),
+                            sourceIP,
+                            pkt.getSenderHardwareAddress()), false);
+                }
+            } else {
+                /*
+                 * Target MAC has been changed. For now, discard it.
+                 * TODO: We may need to send unicast ARP REQUEST on behalf of
+                 * the target back to the sender to trigger the sender to update
+                 * its table
+                 */
+            }
         }
     }
 
@@ -314,19 +335,21 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         if (subnet.isFlatLayer2()) {
             nodeConnectors = new HashSet<NodeConnector>();
             for (Node n : this.switchManager.getNodes()) {
-                nodeConnectors.addAll(this.switchManager
-                        .getUpNodeConnectors(n));
+                nodeConnectors.addAll(this.switchManager.getUpNodeConnectors(n));
             }
         } else {
             nodeConnectors = subnet.getNodeConnectors();
         }
+
         for (NodeConnector p : nodeConnectors) {
-            if (topologyManager.isInternal(p)) {
+
+            //fiter out any non-local or internal ports
+            if (! connectionManager.isLocal(p.getNode()) || topologyManager.isInternal(p)) {
                 continue;
             }
             ARP arp = new ARP();
             byte[] senderIP = subnet.getNetworkAddress().getAddress();
-            byte[] targetIPB = targetIP.getAddress();
+            byte[] targetIPByte = targetIP.getAddress();
             arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
                .setProtocolType(EtherTypes.IPv4.shortValue())
                .setHardwareAddressLength((byte) 6)
@@ -334,14 +357,13 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
                .setOpCode(ARP.REQUEST)
                .setSenderHardwareAddress(getControllerMAC())
                .setSenderProtocolAddress(senderIP)
-               .setTargetHardwareAddress(new byte[] { (byte) 0, (byte) 0,
-                                                     (byte) 0, (byte) 0,
-                                                     (byte) 0, (byte) 0 })
-               .setTargetProtocolAddress(targetIPB);
+               .setTargetHardwareAddress(
+                       new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0 })
+               .setTargetProtocolAddress(targetIPByte);
 
             Ethernet ethernet = new Ethernet();
             ethernet.setSourceMACAddress(getControllerMAC())
-                    .setDestinationMACAddress(new byte[] { (byte) -1,
+                    .setDestinationMACAddress(new byte[] {(byte) -1,
                                                           (byte) -1,
                                                           (byte) -1,
                                                           (byte) -1,
@@ -365,19 +387,10 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
      * The sender MAC is the controller's MAC
      */
     protected void sendUcastARPRequest(HostNodeConnector host, Subnet subnet) {
-        //Long swID = host.getnodeconnectornodeId();
-        //Short portID = host.getnodeconnectorportId();
-        //Node n = NodeCreator.createOFNode(swID);
-        Node n = host.getnodeconnectorNode();
-        if (n == null) {
-            logger.error("cannot send UcastARP because cannot extract node "
-                    + "from HostNodeConnector: {}", host);
-            return;
-        }
+
         NodeConnector outPort = host.getnodeConnector();
         if (outPort == null) {
-            logger.error("cannot send UcastARP because cannot extract "
-                    + "outPort from HostNodeConnector: {}", host);
+            log.error("Failed sending UcastARP because cannot extract output port from Host: {}", host);
             return;
         }
 
@@ -385,18 +398,21 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         byte[] targetIP = host.getNetworkAddress().getAddress();
         byte[] targetMAC = host.getDataLayerAddressBytes();
         ARP arp = new ARP();
-        arp.setHardwareType(ARP.HW_TYPE_ETHERNET).setProtocolType(
-                EtherTypes.IPv4.shortValue())
-                .setHardwareAddressLength((byte) 6).setProtocolAddressLength(
-                        (byte) 4).setOpCode(ARP.REQUEST)
-                .setSenderHardwareAddress(getControllerMAC())
-                .setSenderProtocolAddress(senderIP).setTargetHardwareAddress(
-                        targetMAC).setTargetProtocolAddress(targetIP);
+        arp.setHardwareType(ARP.HW_TYPE_ETHERNET)
+            .setProtocolType(EtherTypes.IPv4.shortValue())
+            .setHardwareAddressLength((byte) 6)
+            .setProtocolAddressLength((byte) 4)
+            .setOpCode(ARP.REQUEST)
+            .setSenderHardwareAddress(getControllerMAC())
+            .setSenderProtocolAddress(senderIP)
+            .setTargetHardwareAddress(targetMAC)
+            .setTargetProtocolAddress(targetIP);
 
         Ethernet ethernet = new Ethernet();
         ethernet.setSourceMACAddress(getControllerMAC())
-                .setDestinationMACAddress(targetMAC).setEtherType(
-                        EtherTypes.ARP.shortValue()).setPayload(arp);
+                .setDestinationMACAddress(targetMAC)
+                .setEtherType(EtherTypes.ARP.shortValue())
+                .setPayload(arp);
 
         RawPacket destPkt = this.dataPacketService.encodeDataPacket(ethernet);
         destPkt.setOutgoingNodeConnector(outPort);
@@ -405,27 +421,26 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
     }
 
     public void find(InetAddress networkAddress) {
-        logger.debug("Received find IP {}", networkAddress);
+        log.trace("Received find IP {}", networkAddress);
 
         Subnet subnet = null;
         if (switchManager != null) {
             subnet = switchManager.getSubnetByNetworkAddress(networkAddress);
         }
         if (subnet == null) {
-            logger.debug("can't find subnet matching IP {}", networkAddress);
+            log.debug("Can't find subnet matching IP {}", networkAddress);
             return;
         }
-        logger.debug("found subnet {}", subnet);
 
-        // send a broadcast ARP Request to this interface
-        sendBcastARPRequest(networkAddress, subnet);
+        // send a broadcast ARP Request to this IP
+        arpRequestReplyEvent.put(new ARPRequest(networkAddress, subnet), false);
     }
 
     /*
      * Probe the host by sending a unicast ARP Request to the host
      */
     public void probe(HostNodeConnector host) {
-        logger.debug("Received probe host {}", host);
+        log.trace("Received probe host {}", host);
 
         Subnet subnet = null;
         if (switchManager != null) {
@@ -433,11 +448,17 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
                     .getNetworkAddress());
         }
         if (subnet == null) {
-            logger.debug("can't find subnet matching {}", host
-                    .getNetworkAddress());
+            log.debug("can't find subnet matching {}", host.getNetworkAddress());
             return;
         }
-        sendUcastARPRequest(host, subnet);
+
+        if (connectionManager.isLocal(host.getnodeconnectorNode())){
+            log.trace("Send a ucast ARP req. to: {}", host);
+            sendUcastARPRequest(host, subnet);
+        } else {
+            log.trace("Raise a ucast ARP req. event to: {}", host);
+            arpRequestReplyEvent.put(new ARPRequest(host, subnet), false);
+        }
     }
 
     /*
@@ -446,12 +467,10 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
      * Need to discover it by sending a Broadcast ARP Request
      */
     protected void handlePuntedIPPacket(IPv4 pkt, NodeConnector p) {
-        InetAddress dIP = null;
-        try {
-            dIP = InetAddress.getByAddress(NetUtils.intToByteArray4(pkt
-                    .getDestinationAddress()));
-        } catch (UnknownHostException e1) {
-            return;
+
+        InetAddress dIP = NetUtils.getInetAddress(pkt.getDestinationAddress());
+        if (dIP == null) {
+           return;
         }
 
         Subnet subnet = null;
@@ -459,14 +478,14 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
             subnet = switchManager.getSubnetByNetworkAddress(dIP);
         }
         if (subnet == null) {
-            logger.debug("can't find subnet matching {}, drop packet", dIP);
+            log.debug("Can't find subnet matching {}, drop packet", dIP);
             return;
         }
-        logger.debug("Found {} matching {}", subnet, dIP);
+        log.trace("Punted IP pkt from {}, sending bcast ARP event...", dIP);
         /*
-         * unknown destination host, initiate ARP request
+         * unknown destination host, initiate bcast ARP request
          */
-        sendBcastARPRequest(dIP, subnet);
+        arpRequestReplyEvent.put(new ARPRequest(dIP, subnet), false);
         return;
     }
 
@@ -485,8 +504,50 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
     void init() {
         arpRequestors = new ConcurrentHashMap<InetAddress, Set<HostNodeConnector>>();
         countDownTimers = new ConcurrentHashMap<InetAddress, Short>();
+
+        allocateCaches();
+        retrieveCaches();
+    }
+
+    @SuppressWarnings({ "unchecked", "deprecation" })
+    private void retrieveCaches() {
+        ConcurrentMap<?,?> map;
+
+        if (this.clusterContainerService == null){
+            log.error("Cluster service unavailable, can't retieve ARPHandler caches!");
+            return;
+        }
+
+        map = clusterContainerService.getCache(ARP_EVENT_CACHE_NAME);
+        if (map != null){
+            this.arpRequestReplyEvent = (ConcurrentMap<ARPEvent, Boolean>) map;
+        } else {
+            log.error("Cache allocation failed for {}", ARP_EVENT_CACHE_NAME);
+        }
     }
 
+    @SuppressWarnings("deprecation")
+    private void allocateCaches() {
+        if (clusterContainerService == null){
+            nonClusterObjectCreate();
+            log.error("Clustering service unavailable. Allocated non-cluster caches for ARPHandler.");
+            return;
+        }
+
+        try{
+            clusterContainerService.createCache(ARP_EVENT_CACHE_NAME,
+                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
+        } catch (CacheConfigException e){
+            log.error("ARPHandler cache configuration invalid!");
+        } catch (CacheExistException e){
+            log.debug("ARPHandler cache exists, skipped allocation.");
+        }
+
+    }
+
+    private void nonClusterObjectCreate(){
+        arpRequestReplyEvent = new ConcurrentHashMap<ARPEvent, Boolean>();
+    }
     /**
      * Function called by the dependency manager when at least one
      * dependency become unsatisfied or when the component is shutting
@@ -512,18 +573,21 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
      * followed by a "destroy ()" calls
      *
      */
-    void stop() {
+    void stop(){
+    }
+
+    void stopping() {
         cancelPeriodicTimer();
     }
 
     void setSwitchManager(ISwitchManager s) {
-        logger.debug("SwitchManager set");
+        log.debug("SwitchManager service set.");
         this.switchManager = s;
     }
 
     void unsetSwitchManager(ISwitchManager s) {
         if (this.switchManager == s) {
-            logger.debug("SwitchManager removed!");
+            log.debug("SwitchManager service UNset.");
             this.switchManager = null;
         }
     }
@@ -533,21 +597,17 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         if (inPkt == null) {
             return PacketResult.IGNORED;
         }
-        logger
-                .trace("Received a frame of size: {}",
-                        inPkt.getPacketData().length);
+        log.trace("Received a frame of size: {}", inPkt.getPacketData().length);
         Packet formattedPak = this.dataPacketService.decodeDataPacket(inPkt);
         if (formattedPak instanceof Ethernet) {
             Object nextPak = formattedPak.getPayload();
             if (nextPak instanceof IPv4) {
-                handlePuntedIPPacket((IPv4) nextPak, inPkt
-                        .getIncomingNodeConnector());
-                logger.trace("Handled IP packet");
-            }
-            if (nextPak instanceof ARP) {
+                log.trace("Handle IP packet: {}", formattedPak);
+                handlePuntedIPPacket((IPv4) nextPak, inPkt.getIncomingNodeConnector());
+            } else if (nextPak instanceof ARP) {
+                log.trace("Handle ARP packet: {}", formattedPak);
                 handleARPPacket((Ethernet) formattedPak, (ARP) nextPak, inPkt
                         .getIncomingNodeConnector());
-                logger.trace("Handled ARP packet");
             }
         }
         return PacketResult.IGNORED;
@@ -556,6 +616,7 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
     private void startPeriodicTimer() {
         this.periodicTimer = new Timer("ArpHandler Periodic Timer");
         this.periodicTimer.scheduleAtFixedRate(new TimerTask() {
+            @SuppressWarnings("deprecation")
             @Override
             public void run() {
                 Set<InetAddress> targetIPs = countDownTimers.keySet();
@@ -569,12 +630,21 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
                         countDownTimers.replace(t, tick);
                     }
                 }
-                for (InetAddress t : expiredTargets) {
-                    countDownTimers.remove(t);
-                    // remove the requestor(s) who have been waited for the ARP
+                for (InetAddress tIP : expiredTargets) {
+                    countDownTimers.remove(tIP);
+                    // Remove the requestor(s) who have been waiting for the ARP
                     // reply from this target for more than 1sec
-                    arpRequestors.remove(t);
-                    logger.debug("{} didn't respond to ARP request", t);
+                    arpRequestors.remove(tIP);
+                    log.debug("ARP reply was not received from {}", tIP);
+                }
+
+                // Clean up ARP event cache
+                try {
+                    if (clusterContainerService.amICoordinator() && ! arpRequestReplyEvent.isEmpty()){
+                        arpRequestReplyEvent.clear();
+                    }
+                } catch (Exception e){
+                    log.warn("ARPHandler: A cluster member failed to clear event cache.");
                 }
             }
         }, 0, 1000);
@@ -593,13 +663,63 @@ public class ArpHandler implements IHostFinder, IListenDataPacket {
         }
         countDownTimers.remove(sourceIP);
         for (HostNodeConnector host : hosts) {
-            logger.debug(
-                    "Sending ARP Reply with src {}/{}, target {}/{}",
-                    new Object[] { sourceMAC, sourceIP,
-                            host.getDataLayerAddressBytes(),
-                            host.getNetworkAddress() });
-            sendARPReply(host.getnodeConnector(), sourceMAC, sourceIP,
-                    host.getDataLayerAddressBytes(), host.getNetworkAddress());
+            log.trace("Sending ARP Reply with src {}/{}, target {}/{}",
+                    new Object[] { sourceMAC, sourceIP, host.getDataLayerAddressBytes(), host.getNetworkAddress() });
+
+            if (connectionManager.isLocal(host.getnodeconnectorNode())){
+                sendARPReply(host.getnodeConnector(),
+                        sourceMAC,
+                        sourceIP,
+                        host.getDataLayerAddressBytes(),
+                        host.getNetworkAddress());
+            } else {
+                arpRequestReplyEvent.put(
+                        new ARPReply(
+                            host.getnodeConnector(),
+                            sourceIP,
+                            sourceMAC,
+                            host.getNetworkAddress(),
+                            host.getDataLayerAddressBytes()), false);
+            }
+        }
+    }
+
+
+    @Override
+    public void entryUpdated(ARPEvent key, Boolean new_value, String cacheName, boolean originLocal) {
+        if (key instanceof ARPRequest) {
+            ARPRequest req = (ARPRequest) key;
+            // If broadcast request
+            if (req.getHost() == null) {
+                sendBcastARPRequest(req.getTargetIP(), req.getSubnet());
+
+            //If unicast and local, send reply
+            } else if (connectionManager.isLocal(req.getHost().getnodeconnectorNode())) {
+                sendUcastARPRequest(req.getHost(), req.getSubnet());
+            }
+        } else if (key instanceof ARPReply) {
+            ARPReply rep = (ARPReply) key;
+            // New reply received by controller, notify all awaiting requestors across the cluster
+            if (new_value) {
+                generateAndSendReply(rep.getTargetIP(), rep.getTargetMac());
+
+            // Otherwise, a specific reply. If local, send out.
+            } else if (connectionManager.isLocal(rep.getPort().getNode())) {
+                sendARPReply(rep.getPort(),
+                        rep.getSourceMac(),
+                        rep.getSourceIP(),
+                        rep.getTargetMac(),
+                        rep.getTargetIP());
+            }
         }
     }
+
+    @Override
+    public void entryCreated(ARPEvent key, String cacheName, boolean originLocal) {
+        // nothing to do
+    }
+    @Override
+    public void entryDeleted(ARPEvent key, String cacheName, boolean originLocal) {
+        // nothing to do
+    }
 }