X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-akka-raft%2Fsrc%2Ftest%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fraft%2Fbehaviors%2FFollowerTest.java;h=b01fd33914f9500c8537cc86032aee924b54f7ed;hp=86caf2a10e6650dd45d9953458127b43adda0620;hb=95d7b8820236d16cb7e37c4a95fcae6f6d55581e;hpb=4f64d27f5434ed7bc82de2b6f6a57ba13ccaec4b diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java index 86caf2a10e..b01fd33914 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/FollowerTest.java @@ -12,18 +12,21 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import akka.actor.ActorRef; import akka.actor.Props; import akka.testkit.TestActorRef; import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.Uninterruptibles; import com.google.protobuf.ByteString; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; @@ -38,6 +41,7 @@ import org.opendaylight.controller.cluster.raft.Snapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.messages.AppendEntries; import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; import org.opendaylight.controller.cluster.raft.messages.InstallSnapshot; @@ -45,10 +49,12 @@ import org.opendaylight.controller.cluster.raft.messages.InstallSnapshotReply; import org.opendaylight.controller.cluster.raft.messages.RaftRPC; import org.opendaylight.controller.cluster.raft.messages.RequestVote; import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply; +import org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload; +import org.opendaylight.controller.cluster.raft.persisted.ServerInfo; import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor; import scala.concurrent.duration.FiniteDuration; -public class FollowerTest extends AbstractRaftActorBehaviorTest { +public class FollowerTest extends AbstractRaftActorBehaviorTest { private final TestActorRef followerActor = actorFactory.createTestActor( Props.create(MessageCollectorActor.class), actorFactory.generateActorId("follower")); @@ -56,7 +62,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { private final TestActorRef leaderActor = actorFactory.createTestActor( Props.create(MessageCollectorActor.class), actorFactory.generateActorId("leader")); - private RaftActorBehavior follower; + private Follower follower; private final short payloadVersion = 5; @@ -71,8 +77,8 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { } @Override - protected RaftActorBehavior createBehavior(RaftActorContext actorContext) { - return new TestFollower(actorContext); + protected Follower createBehavior(RaftActorContext actorContext) { + return spy(new Follower(actorContext)); } @Override @@ -87,38 +93,65 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { return context; } - private static int getElectionTimeoutCount(RaftActorBehavior follower){ - if(follower instanceof TestFollower){ - return ((TestFollower) follower).getElectionTimeoutCount(); - } - return -1; - } - @Test public void testThatAnElectionTimeoutIsTriggered(){ MockRaftActorContext actorContext = createActorContext(); follower = new Follower(actorContext); - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class, + MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class, actorContext.getConfigParams().getElectionTimeOutInterval().$times(6).toMillis()); } @Test - public void testHandleElectionTimeout(){ - logStart("testHandleElectionTimeout"); + public void testHandleElectionTimeoutWhenNoLeaderMessageReceived() { + logStart("testHandleElectionTimeoutWhenNoLeaderMessageReceived"); - follower = new Follower(createActorContext()); + MockRaftActorContext context = createActorContext(); + follower = new Follower(context); - RaftActorBehavior raftBehavior = follower.handleMessage(followerActor, ElectionTimeout.INSTANCE); + Uninterruptibles.sleepUninterruptibly(context.getConfigParams().getElectionTimeOutInterval().toMillis(), + TimeUnit.MILLISECONDS); + RaftActorBehavior raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE); assertTrue(raftBehavior instanceof Candidate); } @Test - public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull(){ + public void testHandleElectionTimeoutWhenLeaderMessageReceived() { + logStart("testHandleElectionTimeoutWhenLeaderMessageReceived"); + + MockRaftActorContext context = createActorContext(); + ((DefaultConfigParamsImpl) context.getConfigParams()). + setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS)); + ((DefaultConfigParamsImpl) context.getConfigParams()).setElectionTimeoutFactor(4); + + follower = new Follower(context); + context.setCurrentBehavior(follower); + + Uninterruptibles.sleepUninterruptibly(context.getConfigParams(). + getElectionTimeOutInterval().toMillis() - 100, TimeUnit.MILLISECONDS); + follower.handleMessage(leaderActor, new AppendEntries(1, "leader", -1, -1, Collections.emptyList(), + -1, -1, (short) 1)); + + Uninterruptibles.sleepUninterruptibly(130, TimeUnit.MILLISECONDS); + RaftActorBehavior raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE); + assertTrue(raftBehavior instanceof Follower); + + Uninterruptibles.sleepUninterruptibly(context.getConfigParams(). + getElectionTimeOutInterval().toMillis() - 150, TimeUnit.MILLISECONDS); + follower.handleMessage(leaderActor, new AppendEntries(1, "leader", -1, -1, Collections.emptyList(), + -1, -1, (short) 1)); + + Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); + raftBehavior = follower.handleMessage(leaderActor, ElectionTimeout.INSTANCE); + assertTrue(raftBehavior instanceof Follower); + } + + @Test + public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull() { logStart("testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNull"); - RaftActorContext context = createActorContext(); + MockRaftActorContext context = createActorContext(); long term = 1000; context.getTermInformation().update(term, null); @@ -130,14 +163,14 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { assertEquals("isVoteGranted", true, reply.isVoteGranted()); assertEquals("getTerm", term, reply.getTerm()); - assertEquals("schedule election", 1, getElectionTimeoutCount(follower)); + verify(follower).scheduleElection(any(FiniteDuration.class)); } @Test public void testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNotTheSameAsCandidateId(){ logStart("testHandleRequestVoteWhenSenderTermEqualToCurrentTermAndVotedForIsNotTheSameAsCandidateId"); - RaftActorContext context = createActorContext(); + MockRaftActorContext context = createActorContext(); long term = 1000; context.getTermInformation().update(term, "test"); @@ -148,7 +181,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { RequestVoteReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, RequestVoteReply.class); assertEquals("isVoteGranted", false, reply.isVoteGranted()); - assertEquals("schedule election", 0, getElectionTimeoutCount(follower)); + verify(follower, never()).scheduleElection(any(FiniteDuration.class)); } @@ -757,7 +790,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { int offset = 0; int snapshotLength = bsSnapshot.size(); int chunkSize = 50; - int totalChunks = (snapshotLength / chunkSize) + ((snapshotLength % chunkSize) > 0 ? 1 : 0); + int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0); int lastIncludedIndex = 1; int chunkIndex = 1; InstallSnapshot lastInstallSnapshot = null; @@ -799,7 +832,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { assertEquals("getFollowerId", context.getId(), reply.getFollowerId()); } - assertNull("Expected null SnapshotTracker", ((Follower) follower).getSnapshotTracker()); + assertNull("Expected null SnapshotTracker", follower.getSnapshotTracker()); } @@ -820,11 +853,11 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { ByteString bsSnapshot = createSnapshot(); int snapshotLength = bsSnapshot.size(); int chunkSize = 50; - int totalChunks = (snapshotLength / chunkSize) + ((snapshotLength % chunkSize) > 0 ? 1 : 0); + int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0); int lastIncludedIndex = 1; // Check that snapshot installation is not in progress - assertNull(((Follower) follower).getSnapshotTracker()); + assertNull(follower.getSnapshotTracker()); // Make sure that we have more than 1 chunk to send assertTrue(totalChunks > 1); @@ -835,22 +868,64 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { chunkData, 1, totalChunks)); // Check if snapshot installation is in progress now - assertNotNull(((Follower) follower).getSnapshotTracker()); + assertNotNull(follower.getSnapshotTracker()); // Send an append entry - AppendEntries appendEntries = mock(AppendEntries.class); - doReturn(context.getTermInformation().getCurrentTerm()).when(appendEntries).getTerm(); + AppendEntries appendEntries = new AppendEntries(1, "leader", 1, 1, + Arrays.asList(newReplicatedLogEntry(2, 1, "3")), 2, -1, (short)1); follower.handleMessage(leaderActor, appendEntries); AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class); - assertEquals(context.getReplicatedLog().lastIndex(), reply.getLogLastIndex()); - assertEquals(context.getReplicatedLog().lastTerm(), reply.getLogLastTerm()); - assertEquals(context.getTermInformation().getCurrentTerm(), reply.getTerm()); + assertEquals("isSuccess", true, reply.isSuccess()); + assertEquals("getLogLastIndex", context.getReplicatedLog().lastIndex(), reply.getLogLastIndex()); + assertEquals("getLogLastTerm", context.getReplicatedLog().lastTerm(), reply.getLogLastTerm()); + assertEquals("getTerm", context.getTermInformation().getCurrentTerm(), reply.getTerm()); + + assertNotNull(follower.getSnapshotTracker()); + } + + @Test + public void testReceivingAppendEntriesDuringInstallSnapshotFromDifferentLeader() throws Exception { + logStart("testReceivingAppendEntriesDuringInstallSnapshotFromDifferentLeader"); + + MockRaftActorContext context = createActorContext(); + + follower = createBehavior(context); + + ByteString bsSnapshot = createSnapshot(); + int snapshotLength = bsSnapshot.size(); + int chunkSize = 50; + int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0); + int lastIncludedIndex = 1; + + // Check that snapshot installation is not in progress + assertNull(follower.getSnapshotTracker()); + + // Make sure that we have more than 1 chunk to send + assertTrue(totalChunks > 1); + + // Send an install snapshot with the first chunk to start the process of installing a snapshot + byte[] chunkData = getNextChunk(bsSnapshot, 0, chunkSize); + follower.handleMessage(leaderActor, new InstallSnapshot(1, "leader", lastIncludedIndex, 1, + chunkData, 1, totalChunks)); - // We should not hit the code that needs to look at prevLogIndex because we are short circuiting - verify(appendEntries, never()).getPrevLogIndex(); + // Check if snapshot installation is in progress now + assertNotNull(follower.getSnapshotTracker()); + + // Send appendEntries with a new term and leader. + AppendEntries appendEntries = new AppendEntries(2, "new-leader", 1, 1, + Arrays.asList(newReplicatedLogEntry(2, 2, "3")), 2, -1, (short)1); + + follower.handleMessage(leaderActor, appendEntries); + + AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class); + assertEquals("isSuccess", true, reply.isSuccess()); + assertEquals("getLogLastIndex", 2, reply.getLogLastIndex()); + assertEquals("getLogLastTerm", 2, reply.getLogLastTerm()); + assertEquals("getTerm", 2, reply.getTerm()); + assertNull(follower.getSnapshotTracker()); } @Test @@ -858,6 +933,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { logStart("testInitialSyncUpWithHandleInstallSnapshot"); MockRaftActorContext context = createActorContext(); + context.setCommitIndex(-1); follower = createBehavior(context); @@ -865,7 +941,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { int offset = 0; int snapshotLength = bsSnapshot.size(); int chunkSize = 50; - int totalChunks = (snapshotLength / chunkSize) + ((snapshotLength % chunkSize) > 0 ? 1 : 0); + int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0); int lastIncludedIndex = 1; int chunkIndex = 1; InstallSnapshot lastInstallSnapshot = null; @@ -927,7 +1003,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { assertEquals("getTerm", 1, reply.getTerm()); assertEquals("getFollowerId", context.getId(), reply.getFollowerId()); - assertNull("Expected null SnapshotTracker", ((Follower) follower).getSnapshotTracker()); + assertNull("Expected null SnapshotTracker", follower.getSnapshotTracker()); } @Test @@ -938,15 +1014,18 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { follower = createBehavior(context); - MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class); + TimeoutNow timeoutNow = MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); assertTrue(elapsed < context.getConfigParams().getElectionTimeOutInterval().toMillis()); + + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), timeoutNow); + assertTrue("Expected Candidate", newBehavior instanceof Candidate); } @Test - public void testFollowerDoesNotScheduleAnElectionIfAutomaticElectionsAreDisabled(){ + public void testFollowerSchedulesElectionIfAutomaticElectionsAreDisabled(){ MockRaftActorContext context = createActorContext(); context.setConfigParams(new DefaultConfigParamsImpl(){ @Override @@ -959,7 +1038,26 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { follower = createBehavior(context); - MessageCollectorActor.assertNoneMatching(followerActor, ElectionTimeout.class, 500); + TimeoutNow timeoutNow = MessageCollectorActor.expectFirstMatching(followerActor, TimeoutNow.class); + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), timeoutNow); + assertSame("handleMessage result", follower, newBehavior); + } + + @Test + public void testFollowerSchedulesElectionIfNonVoting(){ + MockRaftActorContext context = createActorContext(); + context.updatePeerIds(new ServerConfigurationPayload(Arrays.asList(new ServerInfo(context.getId(), false)))); + ((DefaultConfigParamsImpl)context.getConfigParams()).setHeartBeatInterval( + FiniteDuration.apply(100, TimeUnit.MILLISECONDS)); + ((DefaultConfigParamsImpl)context.getConfigParams()).setElectionTimeoutFactor(1); + + follower = new Follower(context, "leader", (short)1); + + ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, + ElectionTimeout.class); + RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout); + assertSame("handleMessage result", follower, newBehavior); + assertNull("Expected null leaderId", follower.getLeaderId()); } @Test @@ -974,7 +1072,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { return 100; } }); - assertEquals("schedule election", 1, getElectionTimeoutCount(follower)); + verify(follower).scheduleElection(any(FiniteDuration.class)); } @Test @@ -982,7 +1080,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { MockRaftActorContext context = createActorContext(); follower = createBehavior(context); follower.handleMessage(leaderActor, "non-raft-rpc"); - assertEquals("schedule election", 0, getElectionTimeoutCount(follower)); + verify(follower, never()).scheduleElection(any(FiniteDuration.class)); } public byte[] getNextChunk (ByteString bs, int offset, int chunkSize){ @@ -992,7 +1090,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { if (chunkSize > snapshotLength) { size = snapshotLength; } else { - if ((start + chunkSize) > snapshotLength) { + if (start + chunkSize > snapshotLength) { size = snapshotLength - start; } } @@ -1039,7 +1137,7 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { } @Override - protected void assertStateChangesToFollowerWhenRaftRPCHasNewerTerm(RaftActorContext actorContext, + protected void assertStateChangesToFollowerWhenRaftRPCHasNewerTerm(MockRaftActorContext actorContext, ActorRef actorRef, RaftRPC rpc) throws Exception { super.assertStateChangesToFollowerWhenRaftRPCHasNewerTerm(actorContext, actorRef, rpc); @@ -1048,28 +1146,9 @@ public class FollowerTest extends AbstractRaftActorBehaviorTest { } @Override - protected void handleAppendEntriesAddSameEntryToLogReply(TestActorRef replyActor) + protected void handleAppendEntriesAddSameEntryToLogReply(final TestActorRef replyActor) throws Exception { AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(replyActor, AppendEntriesReply.class); assertEquals("isSuccess", true, reply.isSuccess()); } - - private static class TestFollower extends Follower { - - int electionTimeoutCount = 0; - - public TestFollower(RaftActorContext context) { - super(context); - } - - @Override - protected void scheduleElection(FiniteDuration interval) { - electionTimeoutCount++; - super.scheduleElection(interval); - } - - public int getElectionTimeoutCount() { - return electionTimeoutCount; - } - } }