BUG 2676 : Add diagnostic message when AppendEntriesReply comes in very late

Log an error message when an AppendEntriesReply arrives beyond the ElectionTimeout
period. This may help detect cases where messages are not being processed
fast enough by the actor system. This particular condition can cause a Leader to
become an IsolatedLeader even if the remote Follower is receiving heartbeats.

Change-Id: Id6a797820b63cb4f18c3d1bbd2781e72674aa9d8
Signed-off-by: Moiz Raja <moraja@cisco.com>

diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java
index b2bb127eab525e35b5f7db5995e9bd57fcbe0746..66b8fba674161ddbc8f9f2e5a64154b62226c683 100644 (file)
--- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java
+++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java
@@ -171,6 +171,14 @@ public abstract class AbstractLeader extends AbstractRaftActorBehavior {
             return this;
         }
 
             return this;
         }
 

+        if(followerLogInformation.timeSinceLastActivity() >
+                context.getConfigParams().getElectionTimeOutInterval().toMillis()) {
+            LOG.error("{} : handleAppendEntriesReply delayed beyond election timeout, " +
+                            "appendEntriesReply : {}, timeSinceLastActivity : {}, lastApplied : {}, commitIndex : {}",
+                    logName(), appendEntriesReply, followerLogInformation.timeSinceLastActivity(),
+                    context.getLastApplied(), context.getCommitIndex());
+        }
+

         followerLogInformation.markFollowerActive();
 
         if (appendEntriesReply.isSuccess()) {
         followerLogInformation.markFollowerActive();
 
         if (appendEntriesReply.isSuccess()) {