From a57559cab8f0dd9204fe4848c85d96659115b63b Mon Sep 17 00:00:00 2001 From: Sai MarapaReddy Date: Wed, 29 Jun 2016 16:31:00 -0700 Subject: [PATCH] Clear leaderId when election timeout occurs in non-voting follower We need to enable election timeouts on non-voting follower and clear the leaderId when it occurs to mimic the behavior when it goes to Candidate on election timeout. Signed-off-by: Sai MarapaReddy Author: Sai MarapaReddy Change-Id: I8b3316e14315a47e09b48af2e3ea16a391ec6c5a Signed-off-by: Tom Pantelis --- .../RaftActorServerConfigurationSupport.java | 4 +++ .../behaviors/AbstractRaftActorBehavior.java | 8 ++--- .../cluster/raft/behaviors/Follower.java | 18 +++++----- ...ftActorServerConfigurationSupportTest.java | 16 +++++++++ .../cluster/raft/behaviors/FollowerTest.java | 33 +++++++++++++++++-- 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java index bbc692e885..87c9b2e3a5 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java @@ -791,6 +791,10 @@ class RaftActorServerConfigurationSupport { @Override void onNewLeader(String newLeader) { + if(newLeader == null) { + return; + } + LOG.debug("{}: New leader {} elected", raftContext.getId(), newLeader); timer.cancel(); diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractRaftActorBehavior.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractRaftActorBehavior.java index 98cbd7b381..4651237de7 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractRaftActorBehavior.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractRaftActorBehavior.java @@ -270,11 +270,9 @@ public abstract class AbstractRaftActorBehavior implements RaftActorBehavior { protected void scheduleElection(FiniteDuration interval) { stopElection(); - if(canStartElection()) { - // Schedule an election. When the scheduler triggers an ElectionTimeout message is sent to itself - electionCancel = context.getActorSystem().scheduler().scheduleOnce(interval, context.getActor(), - ElectionTimeout.INSTANCE, context.getActorSystem().dispatcher(), context.getActor()); - } + // Schedule an election. When the scheduler triggers an ElectionTimeout message is sent to itself + electionCancel = context.getActorSystem().scheduler().scheduleOnce(interval, context.getActor(), + ElectionTimeout.INSTANCE, context.getActorSystem().dispatcher(), context.getActor()); } /** diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java index 02b5d7e72c..d484b25626 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/Follower.java @@ -11,8 +11,8 @@ package org.opendaylight.controller.cluster.raft.behaviors; import akka.actor.ActorRef; import akka.japi.Procedure; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import java.util.ArrayList; +import javax.annotation.Nullable; import org.opendaylight.controller.cluster.raft.RaftActorContext; import org.opendaylight.controller.cluster.raft.RaftState; import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; @@ -65,12 +65,10 @@ public class Follower extends AbstractRaftActorBehavior { initialSyncStatusTracker = new SyncStatusTracker(context.getActor(), getId(), SYNC_THRESHOLD); - if(canStartElection()) { - if (context.getPeerIds().isEmpty() && getLeaderId() == null) { - actor().tell(ElectionTimeout.INSTANCE, actor()); - } else { - scheduleElection(electionDuration()); - } + if (context.getPeerIds().isEmpty() && getLeaderId() == null) { + actor().tell(ElectionTimeout.INSTANCE, actor()); + } else { + scheduleElection(electionDuration()); } } @@ -80,8 +78,8 @@ public class Follower extends AbstractRaftActorBehavior { } @VisibleForTesting - protected final void setLeaderId(final String leaderId) { - this.leaderId = Preconditions.checkNotNull(leaderId); + protected final void setLeaderId(@Nullable final String leaderId) { + this.leaderId = leaderId; } @Override @@ -350,6 +348,8 @@ public class Follower extends AbstractRaftActorBehavior { LOG.debug("{}: Received ElectionTimeout - switching to Candidate", logName()); return internalSwitchBehavior(RaftState.Candidate); } else { + setLeaderId(null); + scheduleElection(electionDuration()); return this; } } diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java index 816e5b2170..10ac2d8cb0 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java @@ -1105,6 +1105,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest { DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl(); configParams.setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS)); + configParams.setElectionTimeoutFactor(5); final String node1ID = "node1"; final String node2ID = "node2"; @@ -1120,8 +1121,10 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest { InMemoryJournal.addEntry(node1ID, 1, new UpdateElectionTerm(1, "downNode1")); InMemoryJournal.addEntry(node1ID, 2, persistedServerConfigEntry); + InMemoryJournal.addEntry(node1ID, 3, new ApplyJournalEntries(0)); InMemoryJournal.addEntry(node2ID, 1, new UpdateElectionTerm(1, "downNode2")); InMemoryJournal.addEntry(node2ID, 2, persistedServerConfigEntry); + InMemoryJournal.addEntry(node2ID, 3, new ApplyJournalEntries(0)); TestActorRef node1Collector = actorFactory.createTestActor( MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), @@ -1168,6 +1171,19 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest { ServerChangeReply reply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), ServerChangeReply.class); assertEquals("getStatus", ServerChangeStatus.NO_LEADER, reply.getStatus()); + // Send an AppendEntries so node1 has a leaderId + + MessageCollectorActor.clearMessages(node1Collector); + + long term = node1RaftActor.getRaftActorContext().getTermInformation().getCurrentTerm(); + node1RaftActorRef.tell(new AppendEntries(term, "downNode1", -1L, -1L, + Collections.emptyList(), 0, -1, (short)1), ActorRef.noSender()); + + // Wait for the ElectionTimeout to clear the leaderId. he leaderId must be null so on the + // ChangeServersVotingStatus message, it will try to elect a leader. + + MessageCollectorActor.expectFirstMatching(node1Collector, ElectionTimeout.class); + // Update node2's peer address and send the message again node1RaftActor.setPeerAddress(node2ID, node2RaftActorRef.path().toString()); diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java index c10ff00356..b8be7be2ae 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; @@ -36,6 +37,8 @@ import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl; import org.opendaylight.controller.cluster.raft.MockRaftActorContext; import org.opendaylight.controller.cluster.raft.RaftActorContext; import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; +import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload; +import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload.ServerInfo; import org.opendaylight.controller.cluster.raft.Snapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; @@ -933,15 +936,19 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { follower = createBehavior(context); - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class); + ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, + ElectionTimeout.class); long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); assertTrue(elapsed < context.getConfigParams().getElectionTimeOutInterval().toMillis()); + + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout); + assertTrue("Expected Candidate", newBehavior instanceof Candidate); } @Test - public void testFollowerDoesNotScheduleAnElectionIfAutomaticElectionsAreDisabled(){ + public void testFollowerSchedulesElectionIfAutomaticElectionsAreDisabled(){ MockRaftActorContext context = createActorContext(); context.setConfigParams(new DefaultConfigParamsImpl(){ @Override @@ -954,7 +961,27 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { follower = createBehavior(context); - MessageCollectorActor.assertNoneMatching(followerActor, ElectionTimeout.class, 500); + ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, + ElectionTimeout.class); + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout); + assertSame("handleMessage result", follower, newBehavior); + } + + @Test + public void testFollowerSchedulesElectionIfNonVoting(){ + MockRaftActorContext context = createActorContext(); + context.updatePeerIds(new ServerConfigurationPayload(Arrays.asList(new ServerInfo(context.getId(), false)))); + ((DefaultConfigParamsImpl)context.getConfigParams()).setHeartBeatInterval( + FiniteDuration.apply(100, TimeUnit.MILLISECONDS)); + ((DefaultConfigParamsImpl)context.getConfigParams()).setElectionTimeoutFactor(1); + + follower = new Follower(context, "leader", (short)1); + + ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, + ElectionTimeout.class); + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout); + assertSame("handleMessage result", follower, newBehavior); + assertNull("Expected null leaderId", follower.getLeaderId()); } @Test -- 2.36.6