+ TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ // Leader should install snapshot - capture and verify ApplySnapshot contents
+
+ ApplySnapshot applySnapshot = expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+ @SuppressWarnings("unchecked")
+ List<Object> snapshotState = (List<Object>) MockRaftActor.toObject(applySnapshot.getSnapshot().getState());
+ assertEquals("Snapshot state", snapshotState, leaderRaftActor.getState());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint());
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+ assertEquals("Leader journal last index", 3, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 3, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 3, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
+
+ // Verify ServerConfigurationPayload entry in both followers
+
+ expectFirstMatching(followerActor, ApplyState.class);
+ assertEquals("Follower journal last index", 3, followerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(followerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ assertEquals("New follower journal last index", 3, newFollowerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
+
+ // Verify new server config was applied in both followers
+
+ assertEquals("Follower peers", Sets.newHashSet(LEADER_ID, NEW_SERVER_ID), followerActorContext.getPeerIds());
+
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID, FOLLOWER_ID), newFollowerActorContext.getPeerIds());
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ expectFirstMatching(followerActor, ApplyState.class);
+
+ assertEquals("Follower commit index", 3, followerActorContext.getCommitIndex());
+ assertEquals("Follower last applied index", 3, followerActorContext.getLastApplied());
+ assertEquals("New follower commit index", 3, newFollowerActorContext.getCommitIndex());
+ assertEquals("New follower last applied index", 3, newFollowerActorContext.getLastApplied());
+
+ List<ReplicatedLogImplEntry> persistedLogEntries = InMemoryJournal.get(LEADER_ID, ReplicatedLogImplEntry.class);
+ assertEquals("Leader ReplicatedLogImplEntry entries", 1, persistedLogEntries.size());
+ ReplicatedLogImplEntry logEntry = persistedLogEntries.get(0);
+ assertEquals("Leader ReplicatedLogImplEntry getTerm", 1, logEntry.getTerm());
+ assertEquals("Leader ReplicatedLogImplEntry getIndex", 3, logEntry.getIndex());
+ assertEquals("Leader ReplicatedLogImplEntry getData", ServerConfigurationPayload.class, logEntry.getData().getClass());
+
+ persistedLogEntries = InMemoryJournal.get(NEW_SERVER_ID, ReplicatedLogImplEntry.class);
+ assertEquals("New follower ReplicatedLogImplEntry entries", 1, persistedLogEntries.size());
+ logEntry = persistedLogEntries.get(0);
+ assertEquals("New follower ReplicatedLogImplEntry getTerm", 1, logEntry.getTerm());
+ assertEquals("New follower ReplicatedLogImplEntry getIndex", 3, logEntry.getIndex());
+ assertEquals("New follower ReplicatedLogImplEntry getData", ServerConfigurationPayload.class,
+ logEntry.getData().getClass());
+ }
+
+ @Test
+ public void testAddServerWithNoExistingFollower() throws Exception {
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+ initialActorContext.setCommitIndex(1);
+ initialActorContext.setLastApplied(1);
+ initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().createEntries(
+ 0, 2, 1).build());
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ // Leader should install snapshot - capture and verify ApplySnapshot contents
+
+ ApplySnapshot applySnapshot = expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+ @SuppressWarnings("unchecked")
+ List<Object> snapshotState = (List<Object>) MockRaftActor.toObject(applySnapshot.getSnapshot().getState());
+ assertEquals("Snapshot state", snapshotState, leaderRaftActor.getState());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint());
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ assertEquals("Leader journal last index", 2, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 2, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 2, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(NEW_SERVER_ID));
+
+ // Verify ServerConfigurationPayload entry in the new follower
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ assertEquals("New follower journal last index", 2, newFollowerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ votingServer(NEW_SERVER_ID));
+
+ // Verify new server config was applied in the new follower
+
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID), newFollowerActorContext.getPeerIds());
+ }
+
+ @Test
+ public void testAddServersAsNonVoting() throws Exception {
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), false), testKit.getRef());
+
+ AddServerReply addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint());
+
+ // Verify ServerConfigurationPayload entry in leader's log
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+
+ assertEquals("Leader journal last index", 0, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 0, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 0, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ nonVotingServer(NEW_SERVER_ID));
+
+ // Verify ServerConfigurationPayload entry in the new follower
+
+ expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
+ assertEquals("New follower journal last index", 0, newFollowerActorContext.getReplicatedLog().lastIndex());
+ verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+ nonVotingServer(NEW_SERVER_ID));
+
+ // Verify new server config was applied in the new follower
+
+ assertEquals("New follower peers", Sets.newHashSet(LEADER_ID), newFollowerActorContext.getPeerIds());
+
+ assertNoneMatching(newFollowerCollectorActor, InstallSnapshot.class, 500);
+
+ // Add another non-voting server.
+
+ clearMessages(leaderCollectorActor);
+
+ RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
+ Follower newFollower2 = new Follower(follower2ActorContext);
+ followerActor.underlyingActor().setBehavior(newFollower2);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID2, followerActor.path().toString(), false), testKit.getRef());
+
+ addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+ assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint());
+
+ expectFirstMatching(leaderCollectorActor, ApplyState.class);
+ assertEquals("Leader journal last index", 1, leaderActorContext.getReplicatedLog().lastIndex());
+ assertEquals("Leader commit index", 1, leaderActorContext.getCommitIndex());
+ assertEquals("Leader last applied index", 1, leaderActorContext.getLastApplied());
+ verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(),
+ votingServer(LEADER_ID), nonVotingServer(NEW_SERVER_ID), nonVotingServer(NEW_SERVER_ID2));
+ }
+
+ @Test
+ public void testAddServerWithOperationInProgress() throws Exception {
+ RaftActorContext initialActorContext = new MockRaftActorContext();
+
+ TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+ MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+ initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+ actorFactory.generateActorId(LEADER_ID));
+
+ MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+ RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+ TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
+ RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
+ Follower newFollower2 = new Follower(follower2ActorContext);
+ followerActor.underlyingActor().setBehavior(newFollower2);
+
+ MockNewFollowerRaftActor newFollowerRaftActorInstance = newFollowerRaftActor.underlyingActor();
+ newFollowerRaftActorInstance.setDropMessageOfType(InstallSnapshot.SERIALIZABLE_CLASS);
+
+ leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+ // Wait for leader's install snapshot and capture it
+
+ InstallSnapshot installSnapshot = expectFirstMatching(newFollowerCollectorActor, InstallSnapshot.class);
+
+ // Send a second AddServer - should get queued
+ JavaTestKit testKit2 = new JavaTestKit(getSystem());
+ leaderActor.tell(new AddServer(NEW_SERVER_ID2, followerActor.path().toString(), false), testKit2.getRef());
+
+ // Continue the first AddServer
+ newFollowerRaftActorInstance.setDropMessageOfType(null);
+ newFollowerRaftActor.tell(installSnapshot, leaderActor);
+
+ // Verify both complete successfully
+ AddServerReply addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
+ assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());