OPNFLWPLUG-1086: Reconciliation framework failure when starting cbench tool for the...

Issue:

CSIT job using cbench had consistent failure with the error message
starting starting with a "closed connection ... exiting" message.
The karaf logs indicated the switches getting disconnected because of
"reconciliation framework" failure.

Observation and Fix:

On analyzing the logs it was found that the "flownodereconciliation"
service of the reconciliation framework was throwing an uncaught
exception (ConcurrentModificationException) which had resulted in
the reconciliation framework failure.

This has been fixed and logs have been enhanced to debug similar issues.

Change-Id: I3cbcea2da32c342c83846f2041d82e921d7acf08
Signed-off-by: Gobinath <gobinath@ericsson.com>
(cherry picked from commit 1fdc1a25f9703550c678b4cd83188de26ad9b19d)

diff --git a/applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/FlowNodeReconciliationImpl.java b/applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/FlowNodeReconciliationImpl.java
index 6aff7710ba3e136577ef92f2b36d3fbdcd06d57f..a3b920f4d841e7378c339085869a6de13cf4053d 100644 (file)
--- a/applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/FlowNodeReconciliationImpl.java
+++ b/applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/FlowNodeReconciliationImpl.java
@@ -27,6 +27,7 @@ import java.util.ListIterator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -127,7 +128,7 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
     private final String serviceName;
     private final int priority;
     private final ResultState resultState;
-    private final Map<DeviceInfo, ListenableFuture<Boolean>> futureMap = new HashMap<>();
+    private final Map<DeviceInfo, ListenableFuture<Boolean>> futureMap = new ConcurrentHashMap<>();
 
     private final ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
 

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/lifecycle/ContextChainHolderImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/lifecycle/ContextChainHolderImpl.java
index da6ab74b03428470dcd814148da5acf9e531b434..53f750d83032c0d65c2c7097750cfc1e86394cfb 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/lifecycle/ContextChainHolderImpl.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/lifecycle/ContextChainHolderImpl.java
@@ -370,7 +370,7 @@ public class ContextChainHolderImpl implements ContextChainHolder, MasterChecker
                     contextChain.continueInitializationAfterReconciliation();
                 } else {
                     OF_EVENT_LOG.debug("Reconciliation framework failure for device {}", deviceInfo);
-                    LOG.warn("Reconciliation framework failure for device {}", deviceInfo);
+                    LOG.warn("Reconciliation framework failure for device {} with result {}", deviceInfo, result);
                     destroyContextChain(deviceInfo);
                 }
             }
@@ -379,7 +379,7 @@ public class ContextChainHolderImpl implements ContextChainHolder, MasterChecker
             public void onFailure(final Throwable throwable) {
                 OF_EVENT_LOG.debug("Reconciliation framework failure for device {} with error {}", deviceInfo,
                         throwable.getMessage());
-                LOG.warn("Reconciliation framework failure.");
+                LOG.warn("Reconciliation framework failure.", throwable);
                 destroyContextChain(deviceInfo);
             }
         };