Handle uncaught exceptions from Clustering Services in HostTracker 81/5081/5
authorMaurice Qureshi <maquresh@cisco.com>
Fri, 31 Jan 2014 23:18:44 +0000 (15:18 -0800)
committerMaurice Qureshi <maquresh@cisco.com>
Tue, 4 Feb 2014 22:53:38 +0000 (14:53 -0800)
There are couple of timer tasks in host tracker which use caches
provided by clustening services. If a container is deleted, then
the cache allocated to that container is also removed. Although
TimerTasks do get cancelled when container is deleted, but if it
happens during the execution of timer thread, then the task will
run to completion (even though it was cancelled) and may try to
access a cache which has already been deallocated.

The accessing of this deallocated cache will result in clustering
service exception which needs to be handled by timer tasks

Also adding toString methods to IPHostId and IPMacHostId for more
readable output when diplaying hosttracker data structures

Change-Id: Id1c481360a4ff71b6eb0afab3c030ef011ca7dcd
Signed-off-by: Maurice Qureshi <maquresh@cisco.com>
opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java
opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java
opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java

index 9e2123c..b8b54b4 100644 (file)
@@ -61,4 +61,19 @@ public class IPHostId implements IHostId, Serializable {
         return new IPHostId(addr);
     }
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("IP=[");
+        if (this.ipAddress != null) {
+            builder.append(this.ipAddress.getHostAddress());
+        }
+        builder.append("]");
+        return (builder.toString());
+    }
 }
index 4ab84eb..e10c5d1 100644 (file)
@@ -80,4 +80,24 @@ public class IPMacHostId implements IHostId, Serializable {
         return new IPMacHostId(ip, mac);
     }
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("IP=[");
+        if (this.ipAddress != null) {
+            builder.append(this.ipAddress.getHostAddress());
+        }
+        builder.append("]")
+               .append("MAC=[");
+        if (this.macAddr != null) {
+            builder.append(this.macAddr.toString());
+        }
+        builder.append("]");
+        return (builder.toString());
+    }
 }
index 9cae434..734a392 100644 (file)
@@ -948,6 +948,10 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
         }
     }
 
+    /*
+     * This thread runs every 4 seconds
+     */
+
     class OutStandingARPHandler extends TimerTask {
         @Override
         public void run() {
@@ -955,53 +959,55 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
                 return;
             }
             ARPPending arphost;
-            /* This routine runs every 4 seconds */
-            logger.trace("Number of Entries in ARP Pending/Failed Lists: ARPPendingList = {}, failedARPReqList = {}",
-                    ARPPendingList.size(), failedARPReqList.size());
-            for (Entry<IHostId, ARPPending> entry : ARPPendingList.entrySet()) {
-                arphost = entry.getValue();
-
-                if (hostsDB.containsKey(arphost.getHostId())) {
-                    // this host is already learned, shouldn't be in
-                    // ARPPendingList
-                    // Remove it and continue
-                    logger.warn("Learned Host {} found in ARPPendingList", decodeIPFromId(arphost.getHostId()));
-                    ARPPendingList.remove(entry.getKey());
-                    continue;
-                }
-                if (arphost.getSent_count() < hostRetryCount) {
-                    /*
-                     * No reply has been received of first ARP Req, send the
-                     * next one. Before sending the ARP, check if ARPHandler is
-                     * available or not
-                     */
-                    if (hostFinder == null) {
-                        logger.warn("ARPHandler Services are not available for Outstanding ARPs");
+            try {
+                for (Entry<IHostId, ARPPending> entry : ARPPendingList.entrySet()) {
+                    arphost = entry.getValue();
+
+                    if (hostsDB.containsKey(arphost.getHostId())) {
+                        // this host is already learned, shouldn't be in
+                        // ARPPendingList
+                        // Remove it and continue
+                        logger.warn("Learned Host {} found in ARPPendingList", decodeIPFromId(arphost.getHostId()));
+                        ARPPendingList.remove(entry.getKey());
                         continue;
                     }
-                    for (IHostFinder hf : hostFinder) {
-                        hf.find(decodeIPFromId(arphost.getHostId()));
-                    }
-                    arphost.sent_count++;
-                    logger.debug("ARP Sent from ARPPending List, IP: {}", decodeIPFromId(arphost.getHostId()));
-                } else if (arphost.getSent_count() >= hostRetryCount) {
-                    /*
-                     * ARP requests have been sent without receiving a reply,
-                     * remove this from the pending list
-                     */
-                    ARPPendingList.remove(entry.getKey());
-                    logger.debug("ARP reply not received after multiple attempts, removing from Pending List IP: {}",
-                            decodeIPFromId(arphost.getHostId()));
-                    /*
-                     * Add this host to a different list which will be processed
-                     * on link up events
-                     */
-                    logger.debug("Adding the host to FailedARPReqList IP: {}", decodeIPFromId(arphost.getHostId()));
-                    failedARPReqList.put(entry.getKey(), arphost);
+                    if (arphost.getSent_count() < hostRetryCount) {
+                        /*
+                         * No reply has been received of first ARP Req, send the
+                         * next one. Before sending the ARP, check if ARPHandler
+                         * is available or not
+                         */
+                        if (hostFinder == null) {
+                            logger.warn("ARPHandler Services are not available for Outstanding ARPs");
+                            continue;
+                        }
+                        for (IHostFinder hf : hostFinder) {
+                            hf.find(decodeIPFromId(arphost.getHostId()));
+                        }
+                        arphost.sent_count++;
+                        logger.debug("ARP Sent from ARPPending List, IP: {}", decodeIPFromId(arphost.getHostId()));
+                    } else if (arphost.getSent_count() >= hostRetryCount) {
+                        /*
+                         * ARP requests have been sent without receiving a
+                         * reply, remove this from the pending list
+                         */
+                        ARPPendingList.remove(entry.getKey());
+                        logger.debug(
+                                "ARP reply not received after multiple attempts, removing from Pending List IP: {}",
+                                decodeIPFromId(arphost.getHostId()));
+                        /*
+                         * Add this host to a different list which will be
+                         * processed on link up events
+                         */
+                        logger.debug("Adding the host to FailedARPReqList IP: {}", decodeIPFromId(arphost.getHostId()));
+                        failedARPReqList.put(entry.getKey(), arphost);
 
-                } else {
-                    logger.error("Inavlid arp_sent count for entry: {}", entry);
+                    } else {
+                        logger.error("Inavlid arp_sent count for entry: {}", entry);
+                    }
                 }
+            } catch (IllegalStateException e) {
+                logger.debug("IllegalStateException Received by OutStandingARPHandler from: {}", e.getMessage());
             }
         }
     }
@@ -1009,10 +1015,10 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
     private class ARPRefreshHandler extends TimerTask {
         @Override
         public void run() {
-            if (stopping) {
+            if ((clusterContainerService != null) && !clusterContainerService.amICoordinator()) {
                 return;
             }
-            if ((clusterContainerService != null) && !clusterContainerService.amICoordinator()) {
+            if (stopping) {
                 return;
             }
             if (!hostRefresh) {
@@ -1026,49 +1032,54 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw
                 logger.error("ARPRefreshHandler(): hostsDB is not allocated yet:");
                 return;
             }
-            for (Entry<IHostId, HostNodeConnector> entry : hostsDB.entrySet()) {
-                HostNodeConnector host = entry.getValue();
-                if (host.isStaticHost()) {
-                    /* this host was learned via API3, don't age it out */
-                    continue;
-                }
-
-                short arp_cntdown = host.getArpSendCountDown();
-                arp_cntdown--;
-                if (arp_cntdown > hostRetryCount) {
-                    host.setArpSendCountDown(arp_cntdown);
-                } else if (arp_cntdown <= 0) {
-                    /*
-                     * No ARP Reply received in last 2 minutes, remove this host
-                     * and inform applications
-                     */
-                    removeKnownHost(entry.getKey());
-                    notifyHostLearnedOrRemoved(host, false);
-                } else if (arp_cntdown <= hostRetryCount) {
-                    /*
-                     * Use the services of arphandler to check if host is still
-                     * there
-                     */
-                    if (logger.isTraceEnabled()) {
-                        logger.trace(
-                                "ARP Probing ({}) for {}({})",
-                                new Object[] { arp_cntdown, host.getNetworkAddress().getHostAddress(),
-                                        HexEncode.bytesToHexString(host.getDataLayerAddressBytes()) });
+            try {
+                for (Entry<IHostId, HostNodeConnector> entry : hostsDB.entrySet()) {
+                    HostNodeConnector host = entry.getValue();
+                    if (host.isStaticHost()) {
+                        /* this host was learned via API3, don't age it out */
+                        continue;
                     }
-                    host.setArpSendCountDown(arp_cntdown);
-                    if (hostFinder == null) {
+
+                    short arp_cntdown = host.getArpSendCountDown();
+                    arp_cntdown--;
+                    if (arp_cntdown > hostRetryCount) {
+                        host.setArpSendCountDown(arp_cntdown);
+                    } else if (arp_cntdown <= 0) {
                         /*
-                         * If hostfinder is not available, then can't send the
-                         * probe. However, continue the age out the hosts since
-                         * we don't know if the host is indeed out there or not.
+                         * No ARP Reply received in last 2 minutes, remove this
+                         * host and inform applications
                          */
-                        logger.trace("ARPHandler is not avaialable, can't send the probe");
-                        continue;
-                    }
-                    for (IHostFinder hf : hostFinder) {
-                        hf.probe(host);
+                        removeKnownHost(entry.getKey());
+                        notifyHostLearnedOrRemoved(host, false);
+                    } else if (arp_cntdown <= hostRetryCount) {
+                        /*
+                         * Use the services of arphandler to check if host is
+                         * still there
+                         */
+                        if (logger.isTraceEnabled()) {
+                            logger.trace(
+                                    "ARP Probing ({}) for {}({})",
+                                    new Object[] { arp_cntdown, host.getNetworkAddress().getHostAddress(),
+                                            HexEncode.bytesToHexString(host.getDataLayerAddressBytes()) });
+                        }
+                        host.setArpSendCountDown(arp_cntdown);
+                        if (hostFinder == null) {
+                            /*
+                             * If hostfinder is not available, then can't send
+                             * the probe. However, continue the age out the
+                             * hosts since we don't know if the host is indeed
+                             * out there or not.
+                             */
+                            logger.trace("ARPHandler is not avaialable, can't send the probe");
+                            continue;
+                        }
+                        for (IHostFinder hf : hostFinder) {
+                            hf.probe(host);
+                        }
                     }
                 }
+            } catch (IllegalStateException e) {
+                logger.debug("IllegalStateException  Received by ARPRefreshHandler from: {}", e.getMessage());
             }
         }
     }