+ final ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ // Leader should install snapshot - capture and verify ApplySnapshot contents
+
+ ApplySnapshot applySnapshot = expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+ List<Object> snapshotState = MockRaftActor.fromState(applySnapshot.getSnapshot().getState());
+ assertEquals("Snapshot state", snapshotState, leaderRaftActor.getState());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint().get());
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+ assertEquals("Leader journal last index", 3, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 3, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 3, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
+
+ // Verify ServerConfigurationPayload entry in both followers
+
+ expectFirstMatching(followerActor, ApplyState.class);
+ assertEquals("Follower journal last index", 3, followerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(followerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ assertEquals("New follower journal last index", 3, newFollowerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
+
+ // Verify new server config was applied in both followers
+
+ assertEquals("Follower peers", Sets.newHashSet(LEADER_ID, NEW_SERVER_ID), followerActorContext.getPeerIds());
+
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID, FOLLOWER_ID),
+ newFollowerActorContext.getPeerIds());
+
+ assertEquals("Follower commit index", 3, followerActorContext.getCommitIndex());
+ assertEquals("Follower last applied index", 3, followerActorContext.getLastApplied());
+ assertEquals("New follower commit index", 3, newFollowerActorContext.getCommitIndex());
+ assertEquals("New follower last applied index", 3, newFollowerActorContext.getLastApplied());
+
+ assertEquals("Leader persisted ReplicatedLogImplEntry entries", 0,
+ InMemoryJournal.get(LEADER_ID, SimpleReplicatedLogEntry.class).size());
+ assertEquals("Leader persisted ServerConfigurationPayload entries", 1,
+ InMemoryJournal.get(LEADER_ID, ServerConfigurationPayload.class).size());
+
+ assertEquals("New follower persisted ReplicatedLogImplEntry entries", 0,
+ InMemoryJournal.get(NEW_SERVER_ID, SimpleReplicatedLogEntry.class).size());
+ assertEquals("New follower persisted ServerConfigurationPayload entries", 1,
+ InMemoryJournal.get(NEW_SERVER_ID, ServerConfigurationPayload.class).size());
+
+ LOG.info("testAddServerWithExistingFollower ending");
+ }
+
+ @Test
+ public void testAddServerWithNoExistingFollower() {
+ LOG.info("testAddServerWithNoExistingFollower starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+ initialActorContext.setCommitIndex(1);
+ initialActorContext.setLastApplied(1);
+ initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().createEntries(
+ 0, 2, 1).build());
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ final RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ final ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ // Leader should install snapshot - capture and verify ApplySnapshot contents
+
+ ApplySnapshot applySnapshot = expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+ List<Object> snapshotState = MockRaftActor.fromState(applySnapshot.getSnapshot().getState());
+ assertEquals("Snapshot state", snapshotState, leaderRaftActor.getState());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint().get());
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ assertEquals("Leader journal last index", 2, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 2, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 2, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(NEW_SERVER_ID));
+
+ // Verify ServerConfigurationPayload entry in the new follower
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ assertEquals("New follower journal last index", 2, newFollowerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(NEW_SERVER_ID));
+
+ // Verify new server config was applied in the new follower
+
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID), newFollowerActorContext.getPeerIds());
+
+ LOG.info("testAddServerWithNoExistingFollower ending");
+ }
+
+ @Test
+ public void testAddServersAsNonVoting() {
+ LOG.info("testAddServersAsNonVoting starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ final RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ final ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), false), testKit.getRef());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint().get());
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+
+ assertEquals("Leader journal last index", 0, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 0, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 0, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ nonVotingServer(NEW_SERVER_ID));
+
+ // Verify ServerConfigurationPayload entry in the new follower
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ assertEquals("New follower journal last index", 0, newFollowerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ nonVotingServer(NEW_SERVER_ID));
+
+ // Verify new server config was applied in the new follower
+
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID), newFollowerActorContext.getPeerIds());
+
+ assertNoneMatching(newFollowerCollectorActor, InstallSnapshot.class, 500);
+
+ // Add another non-voting server.
+
+ clearMessages(leaderCollectorActor);
+
+ RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
+ Follower newFollower2 = new Follower(follower2ActorContext);
+ followerActor.underlyingActor().setBehavior(newFollower2);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID2, followerActor.path().toString(), false), testKit.getRef());
+
+ addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", java.util.Optional.of(LEADER_ID), addServerReply.getLeaderHint());
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ assertEquals("Leader journal last index", 1, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 1, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 1, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(),
+ votingServer(LEADER_ID), nonVotingServer(NEW_SERVER_ID), nonVotingServer(NEW_SERVER_ID2));
+
+ LOG.info("testAddServersAsNonVoting ending");
+ }
+
+ @Test
+ public void testAddServerWithOperationInProgress() {
+ LOG.info("testAddServerWithOperationInProgress starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ final RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ final ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
+ Follower newFollower2 = new Follower(follower2ActorContext);
+ followerActor.underlyingActor().setBehavior(newFollower2);
+
+ MockNewFollowerRaftActor newFollowerRaftActorInstance = newFollowerRaftActor.underlyingActor();
+ newFollowerRaftActorInstance.setDropMessageOfType(InstallSnapshot.class);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ // Wait for leader's install snapshot and capture it
+
+ InstallSnapshot installSnapshot = expectFirstMatching(newFollowerCollectorActor, InstallSnapshot.class);
+
+ // Send a second AddServer - should get queued
+ TestKit testKit2 = new TestKit(getSystem());
+ leaderActor.tell(new AddServer(NEW_SERVER_ID2, followerActor.path().toString(), false), testKit2.getRef());
+
+ // Continue the first AddServer
+ newFollowerRaftActorInstance.setDropMessageOfType(null);
+ newFollowerRaftActor.tell(installSnapshot, leaderActor);
+
+ // Verify both complete successfully
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+
+ addServerReply = testKit2.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+
+ // Verify ServerConfigurationPayload entries in leader's log
+
+ expectMatching(leaderCollectorActor, ApplyState.class, 2);
+ assertEquals("Leader journal last index", 1, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 1, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 1, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(),
+ votingServer(LEADER_ID), votingServer(NEW_SERVER_ID), nonVotingServer(NEW_SERVER_ID2));
+
+ // Verify ServerConfigurationPayload entry in the new follower
+
+ expectMatching(newFollowerCollectorActor, ApplyState.class, 2);
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID, NEW_SERVER_ID2),
+ newFollowerActorContext.getPeerIds());
+
+ LOG.info("testAddServerWithOperationInProgress ending");
+ }
+
+ @Test
+ public void testAddServerWithPriorSnapshotInProgress() {
+ LOG.info("testAddServerWithPriorSnapshotInProgress starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ final RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ // Drop commit message for now to delay snapshot completion
+ leaderRaftActor.setDropMessageOfType(String.class);
+
+ leaderActor.tell(new InitiateCaptureSnapshot(), leaderActor);
+
+ Object commitMsg = expectFirstMatching(leaderCollectorActor, COMMIT_MESSAGE_CLASS);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ leaderRaftActor.setDropMessageOfType(null);
+ leaderActor.tell(commitMsg, leaderActor);
+
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint().get());
+
+ expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ assertEquals("Leader journal last index", 0, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 0, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 0, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(NEW_SERVER_ID));
+
+ LOG.info("testAddServerWithPriorSnapshotInProgress ending");
+ }
+
+ @Test
+ public void testAddServerWithPriorSnapshotCompleteTimeout() {
+ LOG.info("testAddServerWithPriorSnapshotCompleteTimeout starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(1);
+
+ // Drop commit message so the snapshot doesn't complete.
+ leaderRaftActor.setDropMessageOfType(COMMIT_MESSAGE_CLASS);
+
+ leaderActor.tell(new InitiateCaptureSnapshot(), leaderActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.TIMEOUT, addServerReply.getStatus());
+
+ assertEquals("Leader peers size", 0, leaderActorContext.getPeerIds().size());
+
+ LOG.info("testAddServerWithPriorSnapshotCompleteTimeout ending");
+ }
+
+ @Test
+ public void testAddServerWithLeaderChangeBeforePriorSnapshotComplete() {
+ LOG.info("testAddServerWithLeaderChangeBeforePriorSnapshotComplete starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+ ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(100);
+
+ final ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ // Drop the commit message so the snapshot doesn't complete yet.
+ leaderRaftActor.setDropMessageOfType(COMMIT_MESSAGE_CLASS);
+
+ leaderActor.tell(new InitiateCaptureSnapshot(), leaderActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ Object commitMsg = expectFirstMatching(leaderCollectorActor, COMMIT_MESSAGE_CLASS);
+
+ // Change the leader behavior to follower
+ leaderActor.tell(new Follower(leaderActorContext), leaderActor);
+
+ // Drop CaptureSnapshotReply in case install snapshot is incorrectly initiated after the prior
+ // snapshot completes. This will prevent the invalid snapshot from completing and fail the
+ // isCapturing assertion below.
+ leaderRaftActor.setDropMessageOfType(CaptureSnapshotReply.class);
+
+ // Complete the prior snapshot - this should be a no-op b/c it's no longer the leader
+ leaderActor.tell(commitMsg, leaderActor);
+
+ leaderActor.tell(new RaftActorServerConfigurationSupport.ServerOperationTimeout(NEW_SERVER_ID), leaderActor);
+
+ AddServerReply addServerReply = testKit.expectMsgClass(Duration.ofSeconds(5), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.NO_LEADER, addServerReply.getStatus());
+
+ assertEquals("Leader peers size", 0, leaderActorContext.getPeerIds().size());
+ assertEquals("isCapturing", false, leaderActorContext.getSnapshotManager().isCapturing());
+
+ LOG.info("testAddServerWithLeaderChangeBeforePriorSnapshotComplete ending");
+ }
+
+ @Test
+ public void testAddServerWithLeaderChangeDuringInstallSnapshot() {
+ LOG.info("testAddServerWithLeaderChangeDuringInstallSnapshot starting");
+
+ setupNewFollower();
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(8);
+
+ ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);