Alleviate premature elections in followers 64/42564/7
authorTom Pantelis <tpanteli@brocade.com>
Tue, 26 Jul 2016 04:07:02 +0000 (00:07 -0400)
committerTom Pantelis <tpanteli@brocade.com>
Tue, 2 Aug 2016 07:55:32 +0000 (03:55 -0400)
If a follower actor is busy or some non-leader messages take longer to process,
leader messages may get backed up enough to cause the election timer to
expire, thereby resulting in an unwanted election and leader disruption. To
alleviate this scenario, I added a Stopwatch to keep the last time a leader
message was received, ie when a leader message is received it restarts
the Stopwatch. When ElectionTimeout is received, it checks if the
elapsed time of the Stopwatch has exceeded the election timeout
interval. Therefore if leader messages were occurring during the
election timeout interval but were delayed, they will be processed
before the ElectionTimeout message and restart the Stopwatch such that the
elapsed time will/should be less than the election timeout interval by the
time ElectionTimeout is received (unless the last leader message happened to
take longer than the election timeout interval).

There are cases where ElectionTimeout is manually sent to force an
election timeout (eg during leadership transfer). In these cases we
don't want to check the Stopwatch so I added an explicit TimeoutNow
message to distinguish the 2 messages.

Change-Id: I6b745288040da2fdcef1d29cb5ffc482c9e66003
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>
12 files changed:
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/base/messages/TimeoutNow.java [new file with mode: 0644]
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Leader.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedCandidateOnStartupElectionScenarioTest.java
opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShardTest.java

index 87c9b2e..b3506ed 100644 (file)
@@ -22,8 +22,8 @@ import java.util.UUID;
 import javax.annotation.Nullable;
 import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload.ServerInfo;
 import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
-import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
 import org.opendaylight.controller.cluster.raft.base.messages.SnapshotComplete;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader;
 import org.opendaylight.controller.cluster.raft.messages.AddServer;
 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
@@ -729,7 +729,7 @@ class RaftActorServerConfigurationSupport {
                 return;
             }
 
-            raftContext.getActor().tell(ElectionTimeout.INSTANCE, raftContext.getActor());
+            raftContext.getActor().tell(TimeoutNow.INSTANCE, raftContext.getActor());
 
             currentOperationState = new WaitingForLeaderElected(changeVotingStatusContext, previousServerConfig);
         }
diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/base/messages/TimeoutNow.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/base/messages/TimeoutNow.java
new file mode 100644 (file)
index 0000000..5a753c2
--- /dev/null
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.raft.base.messages;
+
+import java.io.Serializable;
+
+/**
+ * Message sent to a follower to force an immediate election time out.
+ *
+ * @author Thomas Pantelis
+ */
+public final class TimeoutNow implements Serializable {
+    private static final long serialVersionUID = 1L;
+    public static final TimeoutNow INSTANCE = new TimeoutNow();
+
+    private TimeoutNow() {
+        // Hidden on purpose
+    }
+
+    private Object readResolve() {
+        return INSTANCE;
+    }
+}
index e4d4266..efece88 100644 (file)
@@ -11,7 +11,9 @@ package org.opendaylight.controller.cluster.raft.behaviors;
 import akka.actor.ActorRef;
 import akka.japi.Procedure;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Stopwatch;
 import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
 import org.opendaylight.controller.cluster.raft.RaftActorContext;
 import org.opendaylight.controller.cluster.raft.RaftState;
@@ -20,6 +22,7 @@ import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload;
 import org.opendaylight.controller.cluster.raft.Snapshot;
 import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
 import org.opendaylight.controller.cluster.raft.messages.InstallSnapshot;
@@ -50,6 +53,7 @@ public class Follower extends AbstractRaftActorBehavior {
         }
     };
 
+    private final Stopwatch lastLeaderMessageTimer = Stopwatch.createUnstarted();
     private SnapshotTracker snapshotTracker = null;
     private String leaderId;
     private short leaderPayloadVersion;
@@ -66,7 +70,7 @@ public class Follower extends AbstractRaftActorBehavior {
         initialSyncStatusTracker = new SyncStatusTracker(context.getActor(), getId(), SYNC_THRESHOLD);
 
         if (context.getPeerIds().isEmpty() && getLeaderId() == null) {
-            actor().tell(ElectionTimeout.INSTANCE, actor());
+            actor().tell(TimeoutNow.INSTANCE, actor());
         } else {
             scheduleElection(electionDuration());
         }
@@ -92,6 +96,14 @@ public class Follower extends AbstractRaftActorBehavior {
         this.leaderPayloadVersion = leaderPayloadVersion;
     }
 
+    private void restartLastLeaderMessageTimer() {
+        if (lastLeaderMessageTimer.isRunning()) {
+            lastLeaderMessageTimer.reset();
+        }
+
+        lastLeaderMessageTimer.start();
+    }
+
     private boolean isLogEntryPresent(long index){
         if(context.getReplicatedLog().isInSnapshot(index)) {
             return true;
@@ -343,15 +355,8 @@ public class Follower extends AbstractRaftActorBehavior {
 
     @Override
     public RaftActorBehavior handleMessage(ActorRef sender, Object originalMessage) {
-        if (originalMessage instanceof ElectionTimeout) {
-            if (canStartElection()) {
-                LOG.debug("{}: Received ElectionTimeout - switching to Candidate", logName());
-                return internalSwitchBehavior(RaftState.Candidate);
-            } else {
-                setLeaderId(null);
-                scheduleElection(electionDuration());
-                return this;
-            }
+        if (originalMessage instanceof ElectionTimeout || originalMessage instanceof TimeoutNow) {
+            return handleElectionTimeout(originalMessage);
         }
 
         final Object message = fromSerializableMessage(originalMessage);
@@ -372,19 +377,50 @@ public class Follower extends AbstractRaftActorBehavior {
         }
 
         if (rpc instanceof InstallSnapshot) {
-            InstallSnapshot installSnapshot = (InstallSnapshot) rpc;
-            handleInstallSnapshot(sender, installSnapshot);
+            handleInstallSnapshot(sender, (InstallSnapshot) rpc);
+            restartLastLeaderMessageTimer();
             scheduleElection(electionDuration());
             return this;
         }
 
         if (!(rpc instanceof RequestVote) || canGrantVote((RequestVote) rpc)) {
+            restartLastLeaderMessageTimer();
             scheduleElection(electionDuration());
         }
 
         return super.handleMessage(sender, rpc);
     }
 
+    private RaftActorBehavior handleElectionTimeout(Object message) {
+        // If the message is ElectionTimeout, verify we haven't actually seen a message from the leader
+        // during the election timeout interval. It may that the election timer expired b/c this actor
+        // was busy and messages got delayed, in which case leader messages would be backed up in the
+        // queue but would be processed before the ElectionTimeout message and thus would restart the
+        // lastLeaderMessageTimer.
+        long lastLeaderMessageInterval = lastLeaderMessageTimer.elapsed(TimeUnit.MILLISECONDS);
+        boolean noLeaderMessageReceived = !lastLeaderMessageTimer.isRunning() || lastLeaderMessageInterval >=
+                context.getConfigParams().getElectionTimeOutInterval().toMillis();
+
+        if(canStartElection()) {
+            if(message instanceof TimeoutNow || noLeaderMessageReceived) {
+                LOG.debug("{}: Received {} - switching to Candidate", logName(), message.getClass().getSimpleName());
+                return internalSwitchBehavior(RaftState.Candidate);
+            } else {
+                LOG.debug("{}: Received ElectionTimeout but lastLeaderMessageInterval {} < election timeout",
+                        logName(), lastLeaderMessageInterval);
+                scheduleElection(electionDuration());
+            }
+        } else if(message instanceof ElectionTimeout) {
+            if(noLeaderMessageReceived) {
+                setLeaderId(null);
+            }
+
+            scheduleElection(electionDuration());
+        }
+
+        return this;
+    }
+
     private void handleInstallSnapshot(final ActorRef sender, InstallSnapshot installSnapshot) {
 
         LOG.debug("{}: handleInstallSnapshot: {}", logName(), installSnapshot);
index 77853f3..e9b45a3 100644 (file)
@@ -19,7 +19,7 @@ import org.opendaylight.controller.cluster.raft.FollowerLogInformation;
 import org.opendaylight.controller.cluster.raft.RaftActorContext;
 import org.opendaylight.controller.cluster.raft.RaftActorLeadershipTransferCohort;
 import org.opendaylight.controller.cluster.raft.RaftState;
-import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
 
 /**
@@ -147,9 +147,9 @@ public class Leader extends AbstractLeader {
             // additional AppendEntries with the latest commit index.
             sendAppendEntries(0, false);
 
-            // Now send an ElectionTimeout to the matching follower to immediately start an election.
+            // Now send a TimeoutNow message to the matching follower to immediately start an election.
             ActorSelection followerActor = context.getPeerActorSelection(followerId);
-            followerActor.tell(ElectionTimeout.INSTANCE, context.getActor());
+            followerActor.tell(TimeoutNow.INSTANCE, context.getActor());
 
             LOG.debug("{}: Leader transfer complete", logName());
 
index 10ac2d8..6e431f2 100644 (file)
@@ -41,6 +41,7 @@ import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
 import org.opendaylight.controller.cluster.raft.base.messages.CaptureSnapshotReply;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
 import org.opendaylight.controller.cluster.raft.base.messages.InitiateCaptureSnapshot;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.base.messages.UpdateElectionTerm;
 import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader;
 import org.opendaylight.controller.cluster.raft.behaviors.Follower;
@@ -1392,7 +1393,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         MessageCollectorActor.expectFirstMatching(node2Collector, RequestVote.class);
 
-        node2RaftActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+        node2RaftActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 
         ServerChangeReply reply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), ServerChangeReply.class);
         assertEquals("getStatus", ServerChangeStatus.OK, reply.getStatus());
index a1301ad..87f9580 100644 (file)
@@ -13,7 +13,7 @@ import com.google.common.collect.ImmutableMap;
 import org.junit.Test;
 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
 import org.opendaylight.controller.cluster.raft.RaftState;
-import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
@@ -76,7 +76,7 @@ public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionS
         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
         member3Actor.expectMessageClass(AppendEntriesReply.class, 2);
 
-        member3ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+        member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 
         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
 
@@ -154,7 +154,7 @@ public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionS
 
         member3Actor.dropMessagesToBehavior(RequestVote.class);
 
-        member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+        member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 
         member1Actor.waitForExpectedMessages(RequestVote.class);
         member3Actor.waitForExpectedMessages(RequestVote.class);
index b8be7be..b11888f 100644 (file)
@@ -24,9 +24,11 @@ import akka.actor.ActorRef;
 import akka.actor.Props;
 import akka.testkit.TestActorRef;
 import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.Uninterruptibles;
 import com.google.protobuf.ByteString;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -43,6 +45,7 @@ import org.opendaylight.controller.cluster.raft.Snapshot;
 import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
 import org.opendaylight.controller.cluster.raft.messages.InstallSnapshot;
@@ -97,23 +100,57 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest<Follower> {
         MockRaftActorContext actorContext = createActorContext();
         follower = new Follower(actorContext);
 
-        MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class,
+        MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class,
                 actorContext.getConfigParams().getElectionTimeOutInterval().$times(6).toMillis());
     }
 
     @Test
-    public void testHandleElectionTimeout(){
-        logStart("testHandleElectionTimeout");
+    public void testHandleElectionTimeoutWhenNoLeaderMessageReceived() {
+        logStart("testHandleElectionTimeoutWhenNoLeaderMessageReceived");
 
-        follower = new Follower(createActorContext());
+        MockRaftActorContext context = createActorContext();
+        follower = new Follower(context);
 
-        RaftActorBehavior raftBehavior = follower.handleMessage(followerActor, ElectionTimeout.INSTANCE);
+        Uninterruptibles.sleepUninterruptibly(context.getConfigParams().getElectionTimeOutInterval().toMillis(),
+                TimeUnit.MILLISECONDS);
+        RaftActorBehavior raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE);
 
         assertTrue(raftBehavior instanceof Candidate);
     }
 
     @Test
-    public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull(){
+    public void testHandleElectionTimeoutWhenLeaderMessageReceived() {
+        logStart("testHandleElectionTimeoutWhenLeaderMessageReceived");
+
+        MockRaftActorContext context = createActorContext();
+        ((DefaultConfigParamsImpl) context.getConfigParams()).
+                setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS));
+        ((DefaultConfigParamsImpl) context.getConfigParams()).setElectionTimeoutFactor(4);
+
+        follower = new Follower(context);
+        context.setCurrentBehavior(follower);
+
+        Uninterruptibles.sleepUninterruptibly(context.getConfigParams().
+                getElectionTimeOutInterval().toMillis() - 100, TimeUnit.MILLISECONDS);
+        follower.handleMessage(leaderActor, new AppendEntries(1, "leader", -1, -1, Collections.emptyList(),
+                -1, -1, (short) 1));
+
+        Uninterruptibles.sleepUninterruptibly(130, TimeUnit.MILLISECONDS);
+        RaftActorBehavior raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE);
+        assertTrue(raftBehavior instanceof Follower);
+
+        Uninterruptibles.sleepUninterruptibly(context.getConfigParams().
+                getElectionTimeOutInterval().toMillis() - 150, TimeUnit.MILLISECONDS);
+        follower.handleMessage(leaderActor, new AppendEntries(1, "leader", -1, -1, Collections.emptyList(),
+                -1, -1, (short) 1));
+
+        Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+        raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE);
+        assertTrue(raftBehavior instanceof Follower);
+    }
+
+    @Test
+    public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull() {
         logStart("testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull");
 
         MockRaftActorContext context = createActorContext();
@@ -936,14 +973,13 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest<Follower> {
 
         follower = createBehavior(context);
 
-        ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor,
-                ElectionTimeout.class);
+        TimeoutNow timeoutNow = MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class);
 
         long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
 
         assertTrue(elapsed < context.getConfigParams().getElectionTimeOutInterval().toMillis());
 
-        RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout);
+        RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), timeoutNow);
         assertTrue("Expected Candidate", newBehavior instanceof Candidate);
     }
 
@@ -961,9 +997,8 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest<Follower> {
 
         follower = createBehavior(context);
 
-        ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor,
-                ElectionTimeout.class);
-        RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout);
+        TimeoutNow timeoutNow = MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class);
+        RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), timeoutNow);
         assertSame("handleMessage result", follower, newBehavior);
     }
 
index b9bd3cd..461dfe5 100644 (file)
@@ -52,6 +52,7 @@ import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
 import org.opendaylight.controller.cluster.raft.base.messages.Replicate;
 import org.opendaylight.controller.cluster.raft.base.messages.SendHeartBeat;
 import org.opendaylight.controller.cluster.raft.base.messages.SendInstallSnapshot;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader.FollowerToSnapshot;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
@@ -2076,7 +2077,7 @@ public class LeaderTest extends AbstractLeaderTest<Leader> {
         MessageCollectorActor.expectMatching(followerActor, AppendEntries.class, 2);
 
         // Leader should force an election timeout
-        MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class);
+        MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class);
 
         verify(mockTransferCohort).transferComplete();
     }
@@ -2109,7 +2110,7 @@ public class LeaderTest extends AbstractLeaderTest<Leader> {
         MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
 
         // Leader should force an election timeout
-        MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class);
+        MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class);
 
         verify(mockTransferCohort).transferComplete();
     }
@@ -2146,7 +2147,7 @@ public class LeaderTest extends AbstractLeaderTest<Leader> {
         leader.handleMessage(leaderActor, new AppendEntriesReply(FOLLOWER_ID, 1, true, 1, 1, (short)0));
 
         // Leader should force an election timeout
-        MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class);
+        MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class);
 
         verify(mockTransferCohort).transferComplete();
     }
index a323aa7..082defd 100644 (file)
@@ -17,6 +17,7 @@ import org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockReplica
 import org.opendaylight.controller.cluster.raft.MockRaftActorContext.SimpleReplicatedLog;
 import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
 
@@ -61,7 +62,7 @@ public class PartitionedCandidateOnStartupElectionScenarioTest extends AbstractL
         member3Actor.expectMessageClass(RequestVote.class, 1);
         member3Actor.expectBehaviorStateChange();
 
-        member1ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+        member1ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 
         member2Actor.waitForExpectedMessages(RequestVote.class);
         member3Actor.waitForExpectedMessages(RequestVote.class);
index 11cc1f0..53f5400 100644 (file)
@@ -14,6 +14,7 @@ import org.junit.Test;
 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
 import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
@@ -207,7 +208,7 @@ public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElecti
         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 
-        member3ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+        member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 
         member1Actor.waitForExpectedMessages(RequestVote.class);
         member2Actor.waitForExpectedMessages(RequestVote.class);
@@ -251,7 +252,7 @@ public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElecti
 
         member3Actor.dropMessagesToBehavior(RequestVote.class);
 
-        member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+        member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 
         member1Actor.waitForExpectedMessages(RequestVote.class);
         member3Actor.waitForExpectedMessages(RequestVote.class);
index 9c6eb25..8f7dd8e 100644 (file)
@@ -93,6 +93,7 @@ import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntrie
 import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.client.messages.FindLeader;
 import org.opendaylight.controller.cluster.raft.client.messages.FindLeaderReply;
 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
@@ -2331,7 +2332,7 @@ public class ShardTest extends AbstractShardTest {
                             "akka://test/user/" + followerShardID.toString())).schemaContext(SCHEMA_CONTEXT).props().
                     withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString());
 
-            leaderShard.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+            leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
             String leaderPath = waitUntilLeader(followerShard);
             assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath);
 
@@ -2407,7 +2408,7 @@ public class ShardTest extends AbstractShardTest {
                             "akka://test/user/" + followerShardID.toString())).schemaContext(SCHEMA_CONTEXT).props().
                     withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString());
 
-            leaderShard.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
+            leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
             String leaderPath = waitUntilLeader(followerShard);
             assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath);
 
index 4810413..d3c3d1d 100644 (file)
@@ -64,6 +64,7 @@ import org.opendaylight.controller.cluster.datastore.modification.Modification;
 import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry;
 import org.opendaylight.controller.cluster.raft.TestActorFactory;
 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
@@ -418,7 +419,7 @@ public class EntityOwnershipShardTest extends AbstractEntityOwnershipTest {
                 ImmutableMap.<String, String>builder().put(peerId1.toString(), peer1.path().toString()).
                         put(peerId2.toString(), peer2.path().toString()).build(), LOCAL_MEMBER_NAME, EntityOwnerSelectionStrategyConfig.newBuilder().build()).
                 withDispatcher(Dispatchers.DefaultDispatcherId()), leaderId.toString());
-        leader.tell(ElectionTimeout.INSTANCE, leader);
+        leader.tell(TimeoutNow.INSTANCE, leader);
 
         ShardTestKit.waitUntilLeader(leader);
 
@@ -566,7 +567,7 @@ public class EntityOwnershipShardTest extends AbstractEntityOwnershipTest {
 
         leader.tell(PoisonPill.getInstance(), ActorRef.noSender());
         peer2.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
-        peer2.tell(ElectionTimeout.INSTANCE, peer2);
+        peer2.tell(TimeoutNow.INSTANCE, peer2);
 
         ShardTestKit.waitUntilLeader(peer2);
 
@@ -592,7 +593,7 @@ public class EntityOwnershipShardTest extends AbstractEntityOwnershipTest {
         TestActorRef<EntityOwnershipShard> leader = actorFactory.createTestActor(newShardProps(leaderId,
                 ImmutableMap.<String, String>builder().put(localId.toString(), shard.path().toString()).build(),
                     LOCAL_MEMBER_NAME, EntityOwnerSelectionStrategyConfig.newBuilder().build()).withDispatcher(Dispatchers.DefaultDispatcherId()), leaderId.toString());
-        leader.tell(ElectionTimeout.INSTANCE, leader);
+        leader.tell(TimeoutNow.INSTANCE, leader);
 
         ShardTestKit.waitUntilLeader(leader);
 

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.