From: Maurice Qureshi Date: Fri, 31 Jan 2014 23:18:44 +0000 (-0800) Subject: Handle uncaught exceptions from Clustering Services in HostTracker X-Git-Tag: autorelease-tag-v20140601202136_82eb3f9~530^2 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=ad4199152e5cd75ba2bcfaa96ca3e4485b4d0a56 Handle uncaught exceptions from Clustering Services in HostTracker There are couple of timer tasks in host tracker which use caches provided by clustening services. If a container is deleted, then the cache allocated to that container is also removed. Although TimerTasks do get cancelled when container is deleted, but if it happens during the execution of timer thread, then the task will run to completion (even though it was cancelled) and may try to access a cache which has already been deallocated. The accessing of this deallocated cache will result in clustering service exception which needs to be handled by timer tasks Also adding toString methods to IPHostId and IPMacHostId for more readable output when diplaying hosttracker data structures Change-Id: Id1c481360a4ff71b6eb0afab3c030ef011ca7dcd Signed-off-by: Maurice Qureshi --- diff --git a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java index 9e2123c6cd..b8b54b45ba 100644 --- a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java +++ b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPHostId.java @@ -61,4 +61,19 @@ public class IPHostId implements IHostId, Serializable { return new IPHostId(addr); } + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("IP=["); + if (this.ipAddress != null) { + builder.append(this.ipAddress.getHostAddress()); + } + builder.append("]"); + return (builder.toString()); + } } diff --git a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java index 4ab84eb653..e10c5d1a78 100644 --- a/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java +++ b/opendaylight/hosttracker/api/src/main/java/org/opendaylight/controller/hosttracker/IPMacHostId.java @@ -80,4 +80,24 @@ public class IPMacHostId implements IHostId, Serializable { return new IPMacHostId(ip, mac); } + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("IP=["); + if (this.ipAddress != null) { + builder.append(this.ipAddress.getHostAddress()); + } + builder.append("]") + .append("MAC=["); + if (this.macAddr != null) { + builder.append(this.macAddr.toString()); + } + builder.append("]"); + return (builder.toString()); + } } diff --git a/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java b/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java index 9cae4348b1..734a392bc1 100644 --- a/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java +++ b/opendaylight/hosttracker/implementation/src/main/java/org/opendaylight/controller/hosttracker/internal/HostTracker.java @@ -948,6 +948,10 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw } } + /* + * This thread runs every 4 seconds + */ + class OutStandingARPHandler extends TimerTask { @Override public void run() { @@ -955,53 +959,55 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw return; } ARPPending arphost; - /* This routine runs every 4 seconds */ - logger.trace("Number of Entries in ARP Pending/Failed Lists: ARPPendingList = {}, failedARPReqList = {}", - ARPPendingList.size(), failedARPReqList.size()); - for (Entry entry : ARPPendingList.entrySet()) { - arphost = entry.getValue(); - - if (hostsDB.containsKey(arphost.getHostId())) { - // this host is already learned, shouldn't be in - // ARPPendingList - // Remove it and continue - logger.warn("Learned Host {} found in ARPPendingList", decodeIPFromId(arphost.getHostId())); - ARPPendingList.remove(entry.getKey()); - continue; - } - if (arphost.getSent_count() < hostRetryCount) { - /* - * No reply has been received of first ARP Req, send the - * next one. Before sending the ARP, check if ARPHandler is - * available or not - */ - if (hostFinder == null) { - logger.warn("ARPHandler Services are not available for Outstanding ARPs"); + try { + for (Entry entry : ARPPendingList.entrySet()) { + arphost = entry.getValue(); + + if (hostsDB.containsKey(arphost.getHostId())) { + // this host is already learned, shouldn't be in + // ARPPendingList + // Remove it and continue + logger.warn("Learned Host {} found in ARPPendingList", decodeIPFromId(arphost.getHostId())); + ARPPendingList.remove(entry.getKey()); continue; } - for (IHostFinder hf : hostFinder) { - hf.find(decodeIPFromId(arphost.getHostId())); - } - arphost.sent_count++; - logger.debug("ARP Sent from ARPPending List, IP: {}", decodeIPFromId(arphost.getHostId())); - } else if (arphost.getSent_count() >= hostRetryCount) { - /* - * ARP requests have been sent without receiving a reply, - * remove this from the pending list - */ - ARPPendingList.remove(entry.getKey()); - logger.debug("ARP reply not received after multiple attempts, removing from Pending List IP: {}", - decodeIPFromId(arphost.getHostId())); - /* - * Add this host to a different list which will be processed - * on link up events - */ - logger.debug("Adding the host to FailedARPReqList IP: {}", decodeIPFromId(arphost.getHostId())); - failedARPReqList.put(entry.getKey(), arphost); + if (arphost.getSent_count() < hostRetryCount) { + /* + * No reply has been received of first ARP Req, send the + * next one. Before sending the ARP, check if ARPHandler + * is available or not + */ + if (hostFinder == null) { + logger.warn("ARPHandler Services are not available for Outstanding ARPs"); + continue; + } + for (IHostFinder hf : hostFinder) { + hf.find(decodeIPFromId(arphost.getHostId())); + } + arphost.sent_count++; + logger.debug("ARP Sent from ARPPending List, IP: {}", decodeIPFromId(arphost.getHostId())); + } else if (arphost.getSent_count() >= hostRetryCount) { + /* + * ARP requests have been sent without receiving a + * reply, remove this from the pending list + */ + ARPPendingList.remove(entry.getKey()); + logger.debug( + "ARP reply not received after multiple attempts, removing from Pending List IP: {}", + decodeIPFromId(arphost.getHostId())); + /* + * Add this host to a different list which will be + * processed on link up events + */ + logger.debug("Adding the host to FailedARPReqList IP: {}", decodeIPFromId(arphost.getHostId())); + failedARPReqList.put(entry.getKey(), arphost); - } else { - logger.error("Inavlid arp_sent count for entry: {}", entry); + } else { + logger.error("Inavlid arp_sent count for entry: {}", entry); + } } + } catch (IllegalStateException e) { + logger.debug("IllegalStateException Received by OutStandingARPHandler from: {}", e.getMessage()); } } } @@ -1009,10 +1015,10 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw private class ARPRefreshHandler extends TimerTask { @Override public void run() { - if (stopping) { + if ((clusterContainerService != null) && !clusterContainerService.amICoordinator()) { return; } - if ((clusterContainerService != null) && !clusterContainerService.amICoordinator()) { + if (stopping) { return; } if (!hostRefresh) { @@ -1026,49 +1032,54 @@ public class HostTracker implements IfIptoHost, IfHostListener, ISwitchManagerAw logger.error("ARPRefreshHandler(): hostsDB is not allocated yet:"); return; } - for (Entry entry : hostsDB.entrySet()) { - HostNodeConnector host = entry.getValue(); - if (host.isStaticHost()) { - /* this host was learned via API3, don't age it out */ - continue; - } - - short arp_cntdown = host.getArpSendCountDown(); - arp_cntdown--; - if (arp_cntdown > hostRetryCount) { - host.setArpSendCountDown(arp_cntdown); - } else if (arp_cntdown <= 0) { - /* - * No ARP Reply received in last 2 minutes, remove this host - * and inform applications - */ - removeKnownHost(entry.getKey()); - notifyHostLearnedOrRemoved(host, false); - } else if (arp_cntdown <= hostRetryCount) { - /* - * Use the services of arphandler to check if host is still - * there - */ - if (logger.isTraceEnabled()) { - logger.trace( - "ARP Probing ({}) for {}({})", - new Object[] { arp_cntdown, host.getNetworkAddress().getHostAddress(), - HexEncode.bytesToHexString(host.getDataLayerAddressBytes()) }); + try { + for (Entry entry : hostsDB.entrySet()) { + HostNodeConnector host = entry.getValue(); + if (host.isStaticHost()) { + /* this host was learned via API3, don't age it out */ + continue; } - host.setArpSendCountDown(arp_cntdown); - if (hostFinder == null) { + + short arp_cntdown = host.getArpSendCountDown(); + arp_cntdown--; + if (arp_cntdown > hostRetryCount) { + host.setArpSendCountDown(arp_cntdown); + } else if (arp_cntdown <= 0) { /* - * If hostfinder is not available, then can't send the - * probe. However, continue the age out the hosts since - * we don't know if the host is indeed out there or not. + * No ARP Reply received in last 2 minutes, remove this + * host and inform applications */ - logger.trace("ARPHandler is not avaialable, can't send the probe"); - continue; - } - for (IHostFinder hf : hostFinder) { - hf.probe(host); + removeKnownHost(entry.getKey()); + notifyHostLearnedOrRemoved(host, false); + } else if (arp_cntdown <= hostRetryCount) { + /* + * Use the services of arphandler to check if host is + * still there + */ + if (logger.isTraceEnabled()) { + logger.trace( + "ARP Probing ({}) for {}({})", + new Object[] { arp_cntdown, host.getNetworkAddress().getHostAddress(), + HexEncode.bytesToHexString(host.getDataLayerAddressBytes()) }); + } + host.setArpSendCountDown(arp_cntdown); + if (hostFinder == null) { + /* + * If hostfinder is not available, then can't send + * the probe. However, continue the age out the + * hosts since we don't know if the host is indeed + * out there or not. + */ + logger.trace("ARPHandler is not avaialable, can't send the probe"); + continue; + } + for (IHostFinder hf : hostFinder) { + hf.probe(host); + } } } + } catch (IllegalStateException e) { + logger.debug("IllegalStateException Received by ARPRefreshHandler from: {}", e.getMessage()); } } }