From: Tom Pantelis Date: Tue, 26 Jul 2016 04:07:02 +0000 (-0400) Subject: Alleviate premature elections in followers X-Git-Tag: release/boron~31 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=364229dd715facec8ef8c73d6c60546c5f38b103 Alleviate premature elections in followers If a follower actor is busy or some non-leader messages take longer to process, leader messages may get backed up enough to cause the election timer to expire, thereby resulting in an unwanted election and leader disruption. To alleviate this scenario, I added a Stopwatch to keep the last time a leader message was received, ie when a leader message is received it restarts the Stopwatch. When ElectionTimeout is received, it checks if the elapsed time of the Stopwatch has exceeded the election timeout interval. Therefore if leader messages were occurring during the election timeout interval but were delayed, they will be processed before the ElectionTimeout message and restart the Stopwatch such that the elapsed time will/should be less than the election timeout interval by the time ElectionTimeout is received (unless the last leader message happened to take longer than the election timeout interval). There are cases where ElectionTimeout is manually sent to force an election timeout (eg during leadership transfer). In these cases we don't want to check the Stopwatch so I added an explicit TimeoutNow message to distinguish the 2 messages. Change-Id: I6b745288040da2fdcef1d29cb5ffc482c9e66003 Signed-off-by: Tom Pantelis --- diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java index 87c9b2e3a5..b3506ed8d0 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java @@ -22,8 +22,8 @@ import java.util.UUID; import javax.annotation.Nullable; import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload.ServerInfo; import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; -import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.SnapshotComplete; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader; import org.opendaylight.controller.cluster.raft.messages.AddServer; import org.opendaylight.controller.cluster.raft.messages.AddServerReply; @@ -729,7 +729,7 @@ class RaftActorServerConfigurationSupport { return; } - raftContext.getActor().tell(ElectionTimeout.INSTANCE, raftContext.getActor()); + raftContext.getActor().tell(TimeoutNow.INSTANCE, raftContext.getActor()); currentOperationState = new WaitingForLeaderElected(changeVotingStatusContext, previousServerConfig); } diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/base/messages/TimeoutNow.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/base/messages/TimeoutNow.java new file mode 100644 index 0000000000..5a753c24c9 --- /dev/null +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/base/messages/TimeoutNow.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.raft.base.messages; + +import java.io.Serializable; + +/** + * Message sent to a follower to force an immediate election time out. + * + * @author Thomas Pantelis + */ +public final class TimeoutNow implements Serializable { + private static final long serialVersionUID = 1L; + public static final TimeoutNow INSTANCE = new TimeoutNow(); + + private TimeoutNow() { + // Hidden on purpose + } + + private Object readResolve() { + return INSTANCE; + } +} diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java index e4d42661de..efece88e28 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java @@ -11,7 +11,9 @@ package org.opendaylight.controller.cluster.raft.behaviors; import akka.actor.ActorRef; import akka.japi.Procedure; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; import java.util.ArrayList; +import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; import org.opendaylight.controller.cluster.raft.RaftActorContext; import org.opendaylight.controller.cluster.raft.RaftState; @@ -20,6 +22,7 @@ import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload; import org.opendaylight.controller.cluster.raft.Snapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntries; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; import org.opendaylight.controller.cluster.raft.messages.InstallSnapshot; @@ -50,6 +53,7 @@ public class Follower extends AbstractRaftActorBehavior { } }; + private final Stopwatch lastLeaderMessageTimer = Stopwatch.createUnstarted(); private SnapshotTracker snapshotTracker = null; private String leaderId; private short leaderPayloadVersion; @@ -66,7 +70,7 @@ public class Follower extends AbstractRaftActorBehavior { initialSyncStatusTracker = new SyncStatusTracker(context.getActor(), getId(), SYNC_THRESHOLD); if (context.getPeerIds().isEmpty() && getLeaderId() == null) { - actor().tell(ElectionTimeout.INSTANCE, actor()); + actor().tell(TimeoutNow.INSTANCE, actor()); } else { scheduleElection(electionDuration()); } @@ -92,6 +96,14 @@ public class Follower extends AbstractRaftActorBehavior { this.leaderPayloadVersion = leaderPayloadVersion; } + private void restartLastLeaderMessageTimer() { + if (lastLeaderMessageTimer.isRunning()) { + lastLeaderMessageTimer.reset(); + } + + lastLeaderMessageTimer.start(); + } + private boolean isLogEntryPresent(long index){ if(context.getReplicatedLog().isInSnapshot(index)) { return true; @@ -343,15 +355,8 @@ public class Follower extends AbstractRaftActorBehavior { @Override public RaftActorBehavior handleMessage(ActorRef sender, Object originalMessage) { - if (originalMessage instanceof ElectionTimeout) { - if (canStartElection()) { - LOG.debug("{}: Received ElectionTimeout - switching to Candidate", logName()); - return internalSwitchBehavior(RaftState.Candidate); - } else { - setLeaderId(null); - scheduleElection(electionDuration()); - return this; - } + if (originalMessage instanceof ElectionTimeout || originalMessage instanceof TimeoutNow) { + return handleElectionTimeout(originalMessage); } final Object message = fromSerializableMessage(originalMessage); @@ -372,19 +377,50 @@ public class Follower extends AbstractRaftActorBehavior { } if (rpc instanceof InstallSnapshot) { - InstallSnapshot installSnapshot = (InstallSnapshot) rpc; - handleInstallSnapshot(sender, installSnapshot); + handleInstallSnapshot(sender, (InstallSnapshot) rpc); + restartLastLeaderMessageTimer(); scheduleElection(electionDuration()); return this; } if (!(rpc instanceof RequestVote) || canGrantVote((RequestVote) rpc)) { + restartLastLeaderMessageTimer(); scheduleElection(electionDuration()); } return super.handleMessage(sender, rpc); } + private RaftActorBehavior handleElectionTimeout(Object message) { + // If the message is ElectionTimeout, verify we haven't actually seen a message from the leader + // during the election timeout interval. It may that the election timer expired b/c this actor + // was busy and messages got delayed, in which case leader messages would be backed up in the + // queue but would be processed before the ElectionTimeout message and thus would restart the + // lastLeaderMessageTimer. + long lastLeaderMessageInterval = lastLeaderMessageTimer.elapsed(TimeUnit.MILLISECONDS); + boolean noLeaderMessageReceived = !lastLeaderMessageTimer.isRunning() || lastLeaderMessageInterval >= + context.getConfigParams().getElectionTimeOutInterval().toMillis(); + + if(canStartElection()) { + if(message instanceof TimeoutNow || noLeaderMessageReceived) { + LOG.debug("{}: Received {} - switching to Candidate", logName(), message.getClass().getSimpleName()); + return internalSwitchBehavior(RaftState.Candidate); + } else { + LOG.debug("{}: Received ElectionTimeout but lastLeaderMessageInterval {} < election timeout", + logName(), lastLeaderMessageInterval); + scheduleElection(electionDuration()); + } + } else if(message instanceof ElectionTimeout) { + if(noLeaderMessageReceived) { + setLeaderId(null); + } + + scheduleElection(electionDuration()); + } + + return this; + } + private void handleInstallSnapshot(final ActorRef sender, InstallSnapshot installSnapshot) { LOG.debug("{}: handleInstallSnapshot: {}", logName(), installSnapshot); diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Leader.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Leader.java index 77853f38c8..e9b45a3e5f 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Leader.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Leader.java @@ -19,7 +19,7 @@ import org.opendaylight.controller.cluster.raft.FollowerLogInformation; import org.opendaylight.controller.cluster.raft.RaftActorContext; import org.opendaylight.controller.cluster.raft.RaftActorLeadershipTransferCohort; import org.opendaylight.controller.cluster.raft.RaftState; -import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; /** @@ -147,9 +147,9 @@ public class Leader extends AbstractLeader { // additional AppendEntries with the latest commit index. sendAppendEntries(0, false); - // Now send an ElectionTimeout to the matching follower to immediately start an election. + // Now send a TimeoutNow message to the matching follower to immediately start an election. ActorSelection followerActor = context.getPeerActorSelection(followerId); - followerActor.tell(ElectionTimeout.INSTANCE, context.getActor()); + followerActor.tell(TimeoutNow.INSTANCE, context.getActor()); LOG.debug("{}: Leader transfer complete", logName()); diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java index 10ac2d8cb0..6e431f28ad 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java @@ -41,6 +41,7 @@ import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; import org.opendaylight.controller.cluster.raft.base.messages.CaptureSnapshotReply; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.InitiateCaptureSnapshot; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.base.messages.UpdateElectionTerm; import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader; import org.opendaylight.controller.cluster.raft.behaviors.Follower; @@ -1392,7 +1393,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest { MessageCollectorActor.expectFirstMatching(node2Collector, RequestVote.class); - node2RaftActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + node2RaftActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); ServerChangeReply reply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), ServerChangeReply.class); assertEquals("getStatus", ServerChangeStatus.OK, reply.getStatus()); diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java index a1301adb7c..87f9580ecd 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java @@ -13,7 +13,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl; import org.opendaylight.controller.cluster.raft.RaftState; -import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; import org.opendaylight.controller.cluster.raft.messages.RequestVote; import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply; @@ -76,7 +76,7 @@ public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionS member3Actor.expectMessageClass(RequestVoteReply.class, 1); member3Actor.expectMessageClass(AppendEntriesReply.class, 2); - member3ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); member3Actor.waitForExpectedMessages(RequestVoteReply.class); @@ -154,7 +154,7 @@ public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionS member3Actor.dropMessagesToBehavior(RequestVote.class); - member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); member1Actor.waitForExpectedMessages(RequestVote.class); member3Actor.waitForExpectedMessages(RequestVote.class); diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java index b8be7be2ae..b11888f76f 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java @@ -24,9 +24,11 @@ import akka.actor.ActorRef; import akka.actor.Props; import akka.testkit.TestActorRef; import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.Uninterruptibles; import com.google.protobuf.ByteString; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; @@ -43,6 +45,7 @@ import org.opendaylight.controller.cluster.raft.Snapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntries; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; import org.opendaylight.controller.cluster.raft.messages.InstallSnapshot; @@ -97,23 +100,57 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { MockRaftActorContext actorContext = createActorContext(); follower = new Follower(actorContext); - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class, + MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class, actorContext.getConfigParams().getElectionTimeOutInterval().$times(6).toMillis()); } @Test - public void testHandleElectionTimeout(){ - logStart("testHandleElectionTimeout"); + public void testHandleElectionTimeoutWhenNoLeaderMessageReceived() { + logStart("testHandleElectionTimeoutWhenNoLeaderMessageReceived"); - follower = new Follower(createActorContext()); + MockRaftActorContext context = createActorContext(); + follower = new Follower(context); - RaftActorBehavior raftBehavior = follower.handleMessage(followerActor, ElectionTimeout.INSTANCE); + Uninterruptibles.sleepUninterruptibly(context.getConfigParams().getElectionTimeOutInterval().toMillis(), + TimeUnit.MILLISECONDS); + RaftActorBehavior raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE); assertTrue(raftBehavior instanceof Candidate); } @Test - public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull(){ + public void testHandleElectionTimeoutWhenLeaderMessageReceived() { + logStart("testHandleElectionTimeoutWhenLeaderMessageReceived"); + + MockRaftActorContext context = createActorContext(); + ((DefaultConfigParamsImpl) context.getConfigParams()). + setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS)); + ((DefaultConfigParamsImpl) context.getConfigParams()).setElectionTimeoutFactor(4); + + follower = new Follower(context); + context.setCurrentBehavior(follower); + + Uninterruptibles.sleepUninterruptibly(context.getConfigParams(). + getElectionTimeOutInterval().toMillis() - 100, TimeUnit.MILLISECONDS); + follower.handleMessage(leaderActor, new AppendEntries(1, "leader", -1, -1, Collections.emptyList(), + -1, -1, (short) 1)); + + Uninterruptibles.sleepUninterruptibly(130, TimeUnit.MILLISECONDS); + RaftActorBehavior raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE); + assertTrue(raftBehavior instanceof Follower); + + Uninterruptibles.sleepUninterruptibly(context.getConfigParams(). + getElectionTimeOutInterval().toMillis() - 150, TimeUnit.MILLISECONDS); + follower.handleMessage(leaderActor, new AppendEntries(1, "leader", -1, -1, Collections.emptyList(), + -1, -1, (short) 1)); + + Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); + raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE); + assertTrue(raftBehavior instanceof Follower); + } + + @Test + public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull() { logStart("testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull"); MockRaftActorContext context = createActorContext(); @@ -936,14 +973,13 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { follower = createBehavior(context); - ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, - ElectionTimeout.class); + TimeoutNow timeoutNow = MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); assertTrue(elapsed < context.getConfigParams().getElectionTimeOutInterval().toMillis()); - RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout); + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), timeoutNow); assertTrue("Expected Candidate", newBehavior instanceof Candidate); } @@ -961,9 +997,8 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { follower = createBehavior(context); - ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, - ElectionTimeout.class); - RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout); + TimeoutNow timeoutNow = MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), timeoutNow); assertSame("handleMessage result", follower, newBehavior); } diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java index b9bd3cd939..461dfe506d 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java @@ -52,6 +52,7 @@ import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.Replicate; import org.opendaylight.controller.cluster.raft.base.messages.SendHeartBeat; import org.opendaylight.controller.cluster.raft.base.messages.SendInstallSnapshot; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader.FollowerToSnapshot; import org.opendaylight.controller.cluster.raft.messages.AppendEntries; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; @@ -2076,7 +2077,7 @@ public class LeaderTest extends AbstractLeaderTest { MessageCollectorActor.expectMatching(followerActor, AppendEntries.class, 2); // Leader should force an election timeout - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class); + MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); verify(mockTransferCohort).transferComplete(); } @@ -2109,7 +2110,7 @@ public class LeaderTest extends AbstractLeaderTest { MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class); // Leader should force an election timeout - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class); + MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); verify(mockTransferCohort).transferComplete(); } @@ -2146,7 +2147,7 @@ public class LeaderTest extends AbstractLeaderTest { leader.handleMessage(leaderActor, new AppendEntriesReply(FOLLOWER_ID, 1, true, 1, 1, (short)0)); // Leader should force an election timeout - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class); + MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); verify(mockTransferCohort).transferComplete(); } diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedCandidateOnStartupElectionScenarioTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedCandidateOnStartupElectionScenarioTest.java index a323aa78ae..082defda94 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedCandidateOnStartupElectionScenarioTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedCandidateOnStartupElectionScenarioTest.java @@ -17,6 +17,7 @@ import org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockReplica import org.opendaylight.controller.cluster.raft.MockRaftActorContext.SimpleReplicatedLog; import org.opendaylight.controller.cluster.raft.RaftState; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.RequestVote; import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply; @@ -61,7 +62,7 @@ public class PartitionedCandidateOnStartupElectionScenarioTest extends AbstractL member3Actor.expectMessageClass(RequestVote.class, 1); member3Actor.expectBehaviorStateChange(); - member1ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + member1ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); member2Actor.waitForExpectedMessages(RequestVote.class); member3Actor.waitForExpectedMessages(RequestVote.class); diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java index 11cc1f0a6d..53f54006ab 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java @@ -14,6 +14,7 @@ import org.junit.Test; import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl; import org.opendaylight.controller.cluster.raft.RaftState; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntries; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; import org.opendaylight.controller.cluster.raft.messages.RequestVote; @@ -207,7 +208,7 @@ public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElecti member3Actor.expectMessageClass(RequestVoteReply.class, 1); member3Actor.expectMessageClass(AppendEntriesReply.class, 1); - member3ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); member1Actor.waitForExpectedMessages(RequestVote.class); member2Actor.waitForExpectedMessages(RequestVote.class); @@ -251,7 +252,7 @@ public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElecti member3Actor.dropMessagesToBehavior(RequestVote.class); - member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); member1Actor.waitForExpectedMessages(RequestVote.class); member3Actor.waitForExpectedMessages(RequestVote.class); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java index 9c6eb257bf..8f7dd8e5b7 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java @@ -93,6 +93,7 @@ import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntrie import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.client.messages.FindLeader; import org.opendaylight.controller.cluster.raft.client.messages.FindLeaderReply; import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState; @@ -2331,7 +2332,7 @@ public class ShardTest extends AbstractShardTest { "akka://test/user/" + followerShardID.toString())).schemaContext(SCHEMA_CONTEXT).props(). withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString()); - leaderShard.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); String leaderPath = waitUntilLeader(followerShard); assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath); @@ -2407,7 +2408,7 @@ public class ShardTest extends AbstractShardTest { "akka://test/user/" + followerShardID.toString())).schemaContext(SCHEMA_CONTEXT).props(). withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString()); - leaderShard.tell(ElectionTimeout.INSTANCE, ActorRef.noSender()); + leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); String leaderPath = waitUntilLeader(followerShard); assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShardTest.java index 48104136b6..d3c3d1d48a 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShardTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShardTest.java @@ -64,6 +64,7 @@ import org.opendaylight.controller.cluster.datastore.modification.Modification; import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; import org.opendaylight.controller.cluster.raft.TestActorFactory; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntries; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; import org.opendaylight.controller.cluster.raft.messages.RequestVote; @@ -418,7 +419,7 @@ public class EntityOwnershipShardTest extends AbstractEntityOwnershipTest { ImmutableMap.builder().put(peerId1.toString(), peer1.path().toString()). put(peerId2.toString(), peer2.path().toString()).build(), LOCAL_MEMBER_NAME, EntityOwnerSelectionStrategyConfig.newBuilder().build()). withDispatcher(Dispatchers.DefaultDispatcherId()), leaderId.toString()); - leader.tell(ElectionTimeout.INSTANCE, leader); + leader.tell(TimeoutNow.INSTANCE, leader); ShardTestKit.waitUntilLeader(leader); @@ -566,7 +567,7 @@ public class EntityOwnershipShardTest extends AbstractEntityOwnershipTest { leader.tell(PoisonPill.getInstance(), ActorRef.noSender()); peer2.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender()); - peer2.tell(ElectionTimeout.INSTANCE, peer2); + peer2.tell(TimeoutNow.INSTANCE, peer2); ShardTestKit.waitUntilLeader(peer2); @@ -592,7 +593,7 @@ public class EntityOwnershipShardTest extends AbstractEntityOwnershipTest { TestActorRef leader = actorFactory.createTestActor(newShardProps(leaderId, ImmutableMap.builder().put(localId.toString(), shard.path().toString()).build(), LOCAL_MEMBER_NAME, EntityOwnerSelectionStrategyConfig.newBuilder().build()).withDispatcher(Dispatchers.DefaultDispatcherId()), leaderId.toString()); - leader.tell(ElectionTimeout.INSTANCE, leader); + leader.tell(TimeoutNow.INSTANCE, leader); ShardTestKit.waitUntilLeader(leader);