Root cause: This problem happens when there are staticRoutes configured but their
next hop IP are unresolved (ie. not responding to ARP Request).
StaticRouteImpl.java spawns a thread requesting hostTracker to discover the host,
this thread is waiting forever due to a bug in the thread (HostTrackerCallable)
that handles the discovery which never returns since the host is not responding.
Solution:
- Instead of creating threads dynamically, we create a threadpool of 2 and reuse
the threads.
- Fix the bug in HostTrackerCallable to sleep(2000) and returns.
Change-Id: Ic04d1dbaf17d5cd6672646050d71abb9885695e9
Signed-off-by: Chi-Vien Ly <chivly@cisco.com>
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Static Routing feature provides the bridge between SDN and Non-SDN networks.
/**
* Static Routing feature provides the bridge between SDN and Non-SDN networks.
*/
public class StaticRoutingImplementation implements IfNewHostNotify,
IForwardingStaticRouting, IObjectReader, IConfigurationContainerAware,
*/
public class StaticRoutingImplementation implements IfNewHostNotify,
IForwardingStaticRouting, IObjectReader, IConfigurationContainerAware,
private IClusterContainerServices clusterContainerService = null;
private Set<IStaticRoutingAware> staticRoutingAware = Collections
.synchronizedSet(new HashSet<IStaticRoutingAware>());
private IClusterContainerServices clusterContainerService = null;
private Set<IStaticRoutingAware> staticRoutingAware = Collections
.synchronizedSet(new HashSet<IStaticRoutingAware>());
+ private ExecutorService executor;
void setStaticRoutingAware(IStaticRoutingAware s) {
if (this.staticRoutingAware != null) {
void setStaticRoutingAware(IStaticRoutingAware s) {
if (this.staticRoutingAware != null) {
- private class NotifyStaticRouteThread extends Thread {
+ private class NotifyStaticRouteWorker implements Callable<Object> {
private StaticRoute staticRoute;
private boolean added;
private StaticRoute staticRoute;
private boolean added;
- public NotifyStaticRouteThread(StaticRoute s, boolean update) {
+ public NotifyStaticRouteWorker(StaticRoute s, boolean update) {
this.staticRoute = s;
this.added = update;
}
this.staticRoute = s;
this.added = update;
}
+ @Override
+ public Object call() throws Exception {
if (!added
|| (staticRoute.getType() == StaticRoute.NextHopType.SWITCHPORT)) {
notifyStaticRouteUpdate(staticRoute, added);
} else {
if (!added
|| (staticRoute.getType() == StaticRoute.NextHopType.SWITCHPORT)) {
notifyStaticRouteUpdate(staticRoute, added);
} else {
- HostNodeConnector host = hostTracker.hostQuery(staticRoute
- .getNextHopAddress());
+ InetAddress nh = staticRoute.getNextHopAddress();
+ HostNodeConnector host = hostTracker.hostQuery(nh);
- Future<HostNodeConnector> future = hostTracker
- .discoverHost(staticRoute.getNextHopAddress());
+ log.debug("Next hop {} is not present, try to discover it", nh.getHostAddress());
+ Future<HostNodeConnector> future = hostTracker.discoverHost(nh);
if (future != null) {
try {
host = future.get();
} catch (Exception e) {
if (future != null) {
try {
host = future.get();
} catch (Exception e) {
}
}
}
if (host != null) {
}
}
}
if (host != null) {
+ log.debug("Next hop {} is found", nh.getHostAddress());
staticRoute.setHost(host);
notifyStaticRouteUpdate(staticRoute, added);
staticRoute.setHost(host);
notifyStaticRouteUpdate(staticRoute, added);
+ } else {
+ log.debug("Next hop {} is still not present, try again later", nh.getHostAddress());
}
}
private void checkAndUpdateListeners(StaticRoute staticRoute, boolean added) {
}
}
private void checkAndUpdateListeners(StaticRoute staticRoute, boolean added) {
- new NotifyStaticRouteThread(staticRoute, added).start();
+ NotifyStaticRouteWorker worker = new NotifyStaticRouteWorker(staticRoute, added);
+ try {
+ executor.submit(worker);
+ } catch (Exception e) {
+ log.error("got Exception ", e);
+ }
}
private void notifyHostUpdate(HostNodeConnector host, boolean added) {
}
private void notifyHostUpdate(HostNodeConnector host, boolean added) {
//staticRoutes = new ConcurrentHashMap<String, StaticRoute>();
allocateCaches();
retrieveCaches();
//staticRoutes = new ConcurrentHashMap<String, StaticRoute>();
allocateCaches();
retrieveCaches();
+ this.executor = Executors.newFixedThreadPool(1);
if (staticRouteConfigs.isEmpty())
loadConfiguration();
if (staticRouteConfigs.isEmpty())
loadConfiguration();
}
@Override
public Status saveConfiguration() {
return this.saveConfig();
}
}
@Override
public Status saveConfiguration() {
return this.saveConfig();
}
private Timer timer;
private Timer arp_refresh_timer;
private String containerName = null;
private Timer timer;
private Timer arp_refresh_timer;
private String containerName = null;
+ private ExecutorService executor;
private static class ARPPending {
protected InetAddress hostIP;
protected short sent_count;
private static class ARPPending {
protected InetAddress hostIP;
protected short sent_count;
timer = new Timer();
timer.schedule(new OutStandingARPHandler(), 4000, 4000);
timer = new Timer();
timer.schedule(new OutStandingARPHandler(), 4000, 4000);
+ executor = Executors.newFixedThreadPool(2);
/* ARP Refresh Timer to go off every 5 seconds to implement ARP aging */
arp_refresh_timer = new Timer();
arp_refresh_timer.schedule(new ARPRefreshHandler(), 5000, 5000);
/* ARP Refresh Timer to go off every 5 seconds to implement ARP aging */
arp_refresh_timer = new Timer();
arp_refresh_timer.schedule(new ARPRefreshHandler(), 5000, 5000);
@Override
public Future<HostNodeConnector> discoverHost(InetAddress networkAddress) {
@Override
public Future<HostNodeConnector> discoverHost(InetAddress networkAddress) {
- ExecutorService executor = Executors.newFixedThreadPool(1);
if (executor == null) {
logger.error("discoverHost: Null executor");
return null;
if (executor == null) {
logger.error("discoverHost: Null executor");
return null;
if (h != null)
return h;
hostTracker.setCallableOnPendingARP(trackedHost, this);
if (h != null)
return h;
hostTracker.setCallableOnPendingARP(trackedHost, this);
+ Thread.sleep(2000); // wait 2sec to see if the host responds
return hostTracker.hostQuery(trackedHost);
}
return hostTracker.hostQuery(trackedHost);
}