Handle uncaught exceptions from Clustering Services in HostTracker

There are couple of timer tasks in host tracker which use caches
provided by clustening services. If a container is deleted, then
the cache allocated to that container is also removed. Although
TimerTasks do get cancelled when container is deleted, but if it
happens during the execution of timer thread, then the task will
run to completion (even though it was cancelled) and may try to
access a cache which has already been deallocated.

The accessing of this deallocated cache will result in clustering
service exception which needs to be handled by timer tasks

Also adding toString methods to IPHostId and IPMacHostId for more
readable output when diplaying hosttracker data structures

Change-Id: Id1c481360a4ff71b6eb0afab3c030ef011ca7dcd
Signed-off-by: Maurice Qureshi <maquresh@cisco.com>

diff --git a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java
index 9e2123c6cd63cfd611f768ec49d2a04785a57864..b8b54b45ba8d10cac4cea25e4f0139bb2ee1c5f1 100644 (file)
--- a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java
+++ b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java
@@ -61,4 +61,19 @@ public class IPHostId implements IHostId, Serializable {
         return new IPHostId(addr);
     }
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("IP=[");
+        if (this.ipAddress != null) {
+            builder.append(this.ipAddress.getHostAddress());
+        }
+        builder.append("]");
+        return (builder.toString());
+    }
 }

diff --git a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java
index 4ab84eb653c02bd563f413c92d32cd0674cb93d1..e10c5d1a78310e5bc1f81393a7899ec4aefab0c2 100644 (file)
--- a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java
+++ b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java
@@ -80,4 +80,24 @@ public class IPMacHostId implements IHostId, Serializable {
         return new IPMacHostId(ip, mac);
     }
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("IP=[");
+        if (this.ipAddress != null) {
+            builder.append(this.ipAddress.getHostAddress());
+        }
+        builder.append("]")
+               .append("MAC=[");
+        if (this.macAddr != null) {
+            builder.append(this.macAddr.toString());
+        }
+        builder.append("]");
+        return (builder.toString());
+    }
 }

diff --git a/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java b/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java
index 9cae4348b12121d7495cb06bec851987bb426b74..734a392bc121f7b33a20408908a3281193aec2aa 100644 (file)
--- a/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java
+++ b/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java
@@ -948,6 +948,10 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
         }
     }
 
+    /*
+     * This thread runs every 4 seconds
+     */
+
     class OutStandingARPHandler extends TimerTask {
         @Override
         public void run() {
@@ -955,53 +959,55 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
                 return;
             }
             ARPPending arphost;
-            /* This routine runs every 4 seconds */
-            logger.trace("Number of Entries in ARP Pending/Failed Lists: ARPPendingList = {}, failedARPReqList = {}",
-                    ARPPendingList.size(), failedARPReqList.size());
-            for (Entry<IHostId, ARPPending> entry : ARPPendingList.entrySet()) {
-                arphost = entry.getValue();
-
-                if (hostsDB.containsKey(arphost.getHostId())) {
-                    // this host is already learned, shouldn't be in
-                    // ARPPendingList
-                    // Remove it and continue
-                    logger.warn("Learned Host {} found in ARPPendingList", decodeIPFromId(arphost.getHostId()));
-                    ARPPendingList.remove(entry.getKey());
-                    continue;
-                }
-                if (arphost.getSent_count() < hostRetryCount) {
-                    /*
-                     * No reply has been received of first ARP Req, send the
-                     * next one. Before sending the ARP, check if ARPHandler is
-                     * available or not
-                     */
-                    if (hostFinder == null) {
-                        logger.warn("ARPHandler Services are not available for Outstanding ARPs");
+            try {
+                for (Entry<IHostId, ARPPending> entry : ARPPendingList.entrySet()) {
+                    arphost = entry.getValue();
+
+                    if (hostsDB.containsKey(arphost.getHostId())) {
+                        // this host is already learned, shouldn't be in
+                        // ARPPendingList
+                        // Remove it and continue
+                        logger.warn("Learned Host {} found in ARPPendingList", decodeIPFromId(arphost.getHostId()));
+                        ARPPendingList.remove(entry.getKey());
                         continue;
                     }
-                    for (IHostFinder hf : hostFinder) {
-                        hf.find(decodeIPFromId(arphost.getHostId()));
-                    }
-                    arphost.sent_count++;
-                    logger.debug("ARP Sent from ARPPending List, IP: {}", decodeIPFromId(arphost.getHostId()));
-                } else if (arphost.getSent_count() >= hostRetryCount) {
-                    /*
-                     * ARP requests have been sent without receiving a reply,
-                     * remove this from the pending list
-                     */
-                    ARPPendingList.remove(entry.getKey());
-                    logger.debug("ARP reply not received after multiple attempts, removing from Pending List IP: {}",
-                            decodeIPFromId(arphost.getHostId()));
-                    /*
-                     * Add this host to a different list which will be processed
-                     * on link up events
-                     */
-                    logger.debug("Adding the host to FailedARPReqList IP: {}", decodeIPFromId(arphost.getHostId()));
-                    failedARPReqList.put(entry.getKey(), arphost);
+                    if (arphost.getSent_count() < hostRetryCount) {
+                        /*
+                         * No reply has been received of first ARP Req, send the
+                         * next one. Before sending the ARP, check if ARPHandler
+                         * is available or not
+                         */
+                        if (hostFinder == null) {
+                            logger.warn("ARPHandler Services are not available for Outstanding ARPs");
+                            continue;
+                        }
+                        for (IHostFinder hf : hostFinder) {
+                            hf.find(decodeIPFromId(arphost.getHostId()));
+                        }
+                        arphost.sent_count++;
+                        logger.debug("ARP Sent from ARPPending List, IP: {}", decodeIPFromId(arphost.getHostId()));
+                    } else if (arphost.getSent_count() >= hostRetryCount) {
+                        /*
+                         * ARP requests have been sent without receiving a
+                         * reply, remove this from the pending list
+                         */
+                        ARPPendingList.remove(entry.getKey());
+                        logger.debug(
+                                "ARP reply not received after multiple attempts, removing from Pending List IP: {}",
+                                decodeIPFromId(arphost.getHostId()));
+                        /*
+                         * Add this host to a different list which will be
+                         * processed on link up events
+                         */
+                        logger.debug("Adding the host to FailedARPReqList IP: {}", decodeIPFromId(arphost.getHostId()));
+                        failedARPReqList.put(entry.getKey(), arphost);
 
-                } else {
-                    logger.error("Inavlid arp_sent count for entry: {}", entry);
+                    } else {
+                        logger.error("Inavlid arp_sent count for entry: {}", entry);
+                    }
                 }
+            } catch (IllegalStateException e) {
+                logger.debug("IllegalStateException Received by OutStandingARPHandler from: {}", e.getMessage());
             }
         }
     }
@@ -1009,10 +1015,10 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
     private class ARPRefreshHandler extends TimerTask {
         @Override
         public void run() {
-            if (stopping) {
+            if ((clusterContainerService != null) && !clusterContainerService.amICoordinator()) {
                 return;
             }
-            if ((clusterContainerService != null) && !clusterContainerService.amICoordinator()) {
+            if (stopping) {
                 return;
             }
             if (!hostRefresh) {
@@ -1026,49 +1032,54 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
                 logger.error("ARPRefreshHandler(): hostsDB is not allocated yet:");
                 return;
             }
-            for (Entry<IHostId, HostNodeConnector> entry : hostsDB.entrySet()) {
-                HostNodeConnector host = entry.getValue();
-                if (host.isStaticHost()) {
-                    /* this host was learned via API3, don't age it out */
-                    continue;
-                }
-
-                short arp_cntdown = host.getArpSendCountDown();
-                arp_cntdown--;
-                if (arp_cntdown > hostRetryCount) {
-                    host.setArpSendCountDown(arp_cntdown);
-                } else if (arp_cntdown <= 0) {
-                    /*
-                     * No ARP Reply received in last 2 minutes, remove this host
-                     * and inform applications
-                     */
-                    removeKnownHost(entry.getKey());
-                    notifyHostLearnedOrRemoved(host, false);
-                } else if (arp_cntdown <= hostRetryCount) {
-                    /*
-                     * Use the services of arphandler to check if host is still
-                     * there
-                     */
-                    if (logger.isTraceEnabled()) {
-                        logger.trace(
-                                "ARP Probing ({}) for {}({})",
-                                new Object[] { arp_cntdown, host.getNetworkAddress().getHostAddress(),
-                                        HexEncode.bytesToHexString(host.getDataLayerAddressBytes()) });
+            try {
+                for (Entry<IHostId, HostNodeConnector> entry : hostsDB.entrySet()) {
+                    HostNodeConnector host = entry.getValue();
+                    if (host.isStaticHost()) {
+                        /* this host was learned via API3, don't age it out */
+                        continue;
                     }
-                    host.setArpSendCountDown(arp_cntdown);
-                    if (hostFinder == null) {
+
+                    short arp_cntdown = host.getArpSendCountDown();
+                    arp_cntdown--;
+                    if (arp_cntdown > hostRetryCount) {
+                        host.setArpSendCountDown(arp_cntdown);
+                    } else if (arp_cntdown <= 0) {
                         /*
-                         * If hostfinder is not available, then can't send the
-                         * probe. However, continue the age out the hosts since
-                         * we don't know if the host is indeed out there or not.
+                         * No ARP Reply received in last 2 minutes, remove this
+                         * host and inform applications
                          */
-                        logger.trace("ARPHandler is not avaialable, can't send the probe");
-                        continue;
-                    }
-                    for (IHostFinder hf : hostFinder) {
-                        hf.probe(host);
+                        removeKnownHost(entry.getKey());
+                        notifyHostLearnedOrRemoved(host, false);
+                    } else if (arp_cntdown <= hostRetryCount) {
+                        /*
+                         * Use the services of arphandler to check if host is
+                         * still there
+                         */
+                        if (logger.isTraceEnabled()) {
+                            logger.trace(
+                                    "ARP Probing ({}) for {}({})",
+                                    new Object[] { arp_cntdown, host.getNetworkAddress().getHostAddress(),
+                                            HexEncode.bytesToHexString(host.getDataLayerAddressBytes()) });
+                        }
+                        host.setArpSendCountDown(arp_cntdown);
+                        if (hostFinder == null) {
+                            /*
+                             * If hostfinder is not available, then can't send
+                             * the probe. However, continue the age out the
+                             * hosts since we don't know if the host is indeed
+                             * out there or not.
+                             */
+                            logger.trace("ARPHandler is not avaialable, can't send the probe");
+                            continue;
+                        }
+                        for (IHostFinder hf : hostFinder) {
+                            hf.probe(host);
+                        }
                     }
                 }
+            } catch (IllegalStateException e) {
+                logger.debug("IllegalStateException  Received by ARPRefreshHandler from: {}", e.getMessage());
             }
         }
     }