Bug 2187: Return OK reply after AddServer persist

The AddServer processing was changed to return OK reply as soon as the
new ServerConfigurationPayload is persisted without waiting for
consensus. Prior, since the new server config is applied immediately in
the leader, if consensus wasn't reached, this would cause the
ShardManager on the calling side to delete new follower actor, resulting
in a "zombie" peer in the leader. Even if consensus isn't reached, the
new server config would've at least most likely been replicated to the
new follower and other down followers would eventually be replicated
when they come back up.

Change-Id: I425fa78d5dd023feda7913ed8d1b5b6c285ccae4
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>

diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java
index 9e40c98e6b079692ef320f1270da9d185a81168c..6c8bf14136d21a218181ffe02aa1cedfa3429de1 100644 (file)
--- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java
+++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupport.java
@@ -18,6 +18,7 @@ import java.util.List;
 import java.util.Queue;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
 import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload.ServerInfo;
 import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
 import org.opendaylight.controller.cluster.raft.base.messages.SnapshotComplete;
@@ -190,22 +191,22 @@ class RaftActorServerConfigurationSupport {
 
             newConfig.add(new ServerInfo(raftContext.getId(), true));
 
-            LOG.debug("{}: New server configuration : {}", raftContext.getId(), newConfig);
+            LOG.debug("{}: Persisting new server configuration : {}", raftContext.getId(), newConfig);
 
             ServerConfigurationPayload payload = new ServerConfigurationPayload(newConfig);
 
             raftActor.persistData(operationContext.getClientRequestor(), operationContext.getContextId(), payload);
 
             currentOperationState = new Persisting(operationContext);
+
+            sendReply(raftActor, operationContext, ServerChangeStatus.OK);
         }
 
         protected void operationComplete(RaftActor raftActor, ServerOperationContext<?> operationContext,
-                ServerChangeStatus status) {
-
-            LOG.debug("{}: Returning {} for operation {}", raftContext.getId(), status, operationContext.getOperation());
-
-            operationContext.getClientRequestor().tell(operationContext.newReply(status, raftActor.getLeaderId()),
-                    raftActor.self());
+                @Nullable ServerChangeStatus replyStatus) {
+            if(replyStatus != null) {
+                sendReply(raftActor, operationContext, replyStatus);
+            }
 
             currentOperationState = IDLE;
 
@@ -215,6 +216,14 @@ class RaftActorServerConfigurationSupport {
             }
         }
 
+        private void sendReply(RaftActor raftActor, ServerOperationContext<?> operationContext,
+                ServerChangeStatus status) {
+            LOG.debug("{}: Returning {} for operation {}", raftContext.getId(), status, operationContext.getOperation());
+
+            operationContext.getClientRequestor().tell(operationContext.newReply(status, raftActor.getLeaderId()),
+                    raftActor.self());
+        }
+
         @Override
         public String toString() {
             return getClass().getSimpleName();
@@ -254,7 +263,7 @@ class RaftActorServerConfigurationSupport {
                 LOG.info("{}: {} has been successfully replicated to a majority of followers",
                         applyState.getReplicatedLogEntry().getData());
 
-                operationComplete(raftActor, operationContext, ServerChangeStatus.OK);
+                operationComplete(raftActor, operationContext, null);
             }
         }
     }

diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/MockRaftActorContext.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/MockRaftActorContext.java
index 5104fde125c518b86e50fe63d68a93cc087051d8..7f70796bbd27520106e2aa544fbc110fce221c9f 100644 (file)
--- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/MockRaftActorContext.java
+++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/MockRaftActorContext.java
@@ -13,13 +13,10 @@ import akka.actor.ActorSelection;
 import akka.actor.ActorSystem;
 import akka.actor.Props;
 import akka.japi.Procedure;
-
 import com.google.protobuf.GeneratedMessage;
-
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
-
 import org.opendaylight.controller.cluster.NonPersistentDataProvider;
 import org.opendaylight.controller.cluster.raft.policy.RaftPolicy;
 import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload;
@@ -67,6 +64,7 @@ public class MockRaftActorContext extends RaftActorContextImpl {
     public MockRaftActorContext(){
         super(null, null, "test", newElectionTerm(), -1, -1, new HashMap<String, String>(),
                 new DefaultConfigParamsImpl(), new NonPersistentDataProvider(), LOG);
+        setReplicatedLog(new MockReplicatedLogBuilder().build());
     }
 
     public MockRaftActorContext(String id, ActorSystem system, ActorRef actor){

diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java
index 3d6f7f414ce7b1f1fc03e876fb7b35912656f23a..3e04e4c1bb396000f3010b507cea6aecc33610c1 100644 (file)
--- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java
+++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorServerConfigurationSupportTest.java
@@ -8,8 +8,10 @@
 package org.opendaylight.controller.cluster.raft;
 
 import static org.junit.Assert.assertEquals;
+import static org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor.assertNoneMatching;
 import static org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor.clearMessages;
 import static org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor.expectFirstMatching;
+import static org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor.expectMatching;
 import akka.actor.ActorRef;
 import akka.actor.Props;
 import akka.actor.UntypedActor;
@@ -133,12 +135,14 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         clearMessages(followerActor);
 
         MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
 
         leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
 
         // Leader should install snapshot - capture and verify ApplySnapshot contents
 
         ApplySnapshot applySnapshot = expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+        @SuppressWarnings("unchecked")
         List<Object> snapshotState = (List<Object>) MockRaftActor.toObject(applySnapshot.getSnapshot().getState());
         assertEquals("Snapshot state", snapshotState, leaderRaftActor.getState());
 
@@ -148,6 +152,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entry in leader's log
 
+        expectFirstMatching(leaderCollectorActor, ApplyState.class);
         RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
         assertEquals("Leader journal last index", 3, leaderActorContext.getReplicatedLog().lastIndex());
         assertEquals("Leader commit index", 3, leaderActorContext.getCommitIndex());
@@ -157,10 +162,12 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entry in both followers
 
+        expectFirstMatching(followerActor, ApplyState.class);
         assertEquals("Follower journal last index", 3, followerActorContext.getReplicatedLog().lastIndex());
         verifyServerConfigurationPayloadEntry(followerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
                 votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
 
+        expectFirstMatching(newFollowerCollectorActor, ApplyState.class);
         assertEquals("New follower journal last index", 3, newFollowerActorContext.getReplicatedLog().lastIndex());
         verifyServerConfigurationPayloadEntry(newFollowerActorContext.getReplicatedLog(), votingServer(LEADER_ID),
                 votingServer(FOLLOWER_ID), votingServer(NEW_SERVER_ID));
@@ -211,11 +218,14 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
         RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
 
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
         leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
 
         // Leader should install snapshot - capture and verify ApplySnapshot contents
 
         ApplySnapshot applySnapshot = expectFirstMatching(newFollowerCollectorActor, ApplySnapshot.class);
+        @SuppressWarnings("unchecked")
         List<Object> snapshotState = (List<Object>) MockRaftActor.toObject(applySnapshot.getSnapshot().getState());
         assertEquals("Snapshot state", snapshotState, leaderRaftActor.getState());
 
@@ -225,6 +235,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entry in leader's log
 
+        expectFirstMatching(leaderCollectorActor, ApplyState.class);
         assertEquals("Leader journal last index", 2, leaderActorContext.getReplicatedLog().lastIndex());
         assertEquals("Leader commit index", 2, leaderActorContext.getCommitIndex());
         assertEquals("Leader last applied index", 2, leaderActorContext.getLastApplied());
@@ -246,9 +257,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServersAsNonVoting() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -258,6 +266,8 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
         RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
 
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
         leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), false), testKit.getRef());
 
         AddServerReply addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
@@ -266,6 +276,8 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entry in leader's log
 
+        expectFirstMatching(leaderCollectorActor, ApplyState.class);
+
         assertEquals("Leader journal last index", 0, leaderActorContext.getReplicatedLog().lastIndex());
         assertEquals("Leader commit index", 0, leaderActorContext.getCommitIndex());
         assertEquals("Leader last applied index", 0, leaderActorContext.getLastApplied());
@@ -283,10 +295,12 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         assertEquals("New follower peers", Sets.newHashSet(LEADER_ID), newFollowerActorContext.getPeerIds());
 
-        MessageCollectorActor.assertNoneMatching(newFollowerCollectorActor, InstallSnapshot.class, 500);
+        assertNoneMatching(newFollowerCollectorActor, InstallSnapshot.class, 500);
 
         // Add another non-voting server.
 
+        clearMessages(leaderCollectorActor);
+
         RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
         Follower newFollower2 = new Follower(follower2ActorContext);
         followerActor.underlyingActor().setBehavior(newFollower2);
@@ -297,6 +311,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
         assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint());
 
+        expectFirstMatching(leaderCollectorActor, ApplyState.class);
         assertEquals("Leader journal last index", 1, leaderActorContext.getReplicatedLog().lastIndex());
         assertEquals("Leader commit index", 1, leaderActorContext.getCommitIndex());
         assertEquals("Leader last applied index", 1, leaderActorContext.getLastApplied());
@@ -307,9 +322,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServerWithOperationInProgress() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -319,6 +331,8 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
         RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
 
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
+
         RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
         Follower newFollower2 = new Follower(follower2ActorContext);
         followerActor.underlyingActor().setBehavior(newFollower2);
@@ -349,6 +363,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entries in leader's log
 
+        expectMatching(leaderCollectorActor, ApplyState.class, 2);
         assertEquals("Leader journal last index", 1, leaderActorContext.getReplicatedLog().lastIndex());
         assertEquals("Leader commit index", 1, leaderActorContext.getCommitIndex());
         assertEquals("Leader last applied index", 1, leaderActorContext.getLastApplied());
@@ -357,8 +372,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entry in the new follower
 
-        MessageCollectorActor.expectMatching(newFollowerCollectorActor, ApplyState.class, 2);
-
+        expectMatching(newFollowerCollectorActor, ApplyState.class, 2);
         assertEquals("New follower peers", Sets.newHashSet(LEADER_ID, NEW_SERVER_ID2),
                newFollowerActorContext.getPeerIds());
     }
@@ -366,9 +380,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServerWithPriorSnapshotInProgress() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -378,10 +389,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
         RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
 
-        TestActorRef<MessageCollectorActor> leaderCollectorActor = actorFactory.createTestActor(
-                MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
-                actorFactory.generateActorId(LEADER_ID + "Collector"));
-        leaderRaftActor.setCollectorActor(leaderCollectorActor);
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
 
         // Drop commit message for now to delay snapshot completion
         leaderRaftActor.setDropMessageOfType(String.class);
@@ -403,6 +411,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         // Verify ServerConfigurationPayload entry in leader's log
 
+        expectFirstMatching(leaderCollectorActor, ApplyState.class);
         assertEquals("Leader journal last index", 0, leaderActorContext.getReplicatedLog().lastIndex());
         assertEquals("Leader commit index", 0, leaderActorContext.getCommitIndex());
         assertEquals("Leader last applied index", 0, leaderActorContext.getLastApplied());
@@ -413,9 +422,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServerWithPriorSnapshotCompleteTimeout() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -427,11 +433,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(1);
 
-        TestActorRef<MessageCollectorActor> leaderCollectorActor = actorFactory.createTestActor(
-                MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
-                actorFactory.generateActorId(LEADER_ID + "Collector"));
-        leaderRaftActor.setCollectorActor(leaderCollectorActor);
-
         // Drop commit message so the snapshot doesn't complete.
         leaderRaftActor.setDropMessageOfType(String.class);
 
@@ -448,9 +449,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServerWithLeaderChangeBeforePriorSnapshotComplete() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -461,10 +459,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
         ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(100);
 
-        TestActorRef<MessageCollectorActor> leaderCollectorActor = actorFactory.createTestActor(
-                MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
-                actorFactory.generateActorId(LEADER_ID + "Collector"));
-        leaderRaftActor.setCollectorActor(leaderCollectorActor);
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
 
         // Drop the commit message so the snapshot doesn't complete yet.
         leaderRaftActor.setDropMessageOfType(String.class);
@@ -498,9 +493,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServerWithLeaderChangeDuringInstallSnapshot() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -512,10 +504,7 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
 
         ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(8);
 
-        TestActorRef<MessageCollectorActor> leaderCollectorActor = actorFactory.createTestActor(
-                MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
-                actorFactory.generateActorId(LEADER_ID + "Collector"));
-        leaderRaftActor.setCollectorActor(leaderCollectorActor);
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
 
         // Drop the UnInitializedFollowerSnapshotReply to delay it.
         leaderRaftActor.setDropMessageOfType(UnInitializedFollowerSnapshotReply.class);
@@ -544,9 +533,6 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
     @Test
     public void testAddServerWithInstallSnapshotTimeout() throws Exception {
         RaftActorContext initialActorContext = new MockRaftActorContext();
-        initialActorContext.setCommitIndex(-1);
-        initialActorContext.setLastApplied(-1);
-        initialActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
 
         TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
                 MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
@@ -588,6 +574,32 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         assertEquals("getStatus", ServerChangeStatus.NO_LEADER, addServerReply.getStatus());
     }
 
+    @Test
+    public void testAddServerWithNoConsensusReached() {
+        RaftActorContext initialActorContext = new MockRaftActorContext();
+
+        TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(
+                MockLeaderRaftActor.props(ImmutableMap.<String, String>of(),
+                        initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()),
+                actorFactory.generateActorId(LEADER_ID));
+
+        MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
+        RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
+
+        newFollowerRaftActor.underlyingActor().setDropMessageOfType(AppendEntries.class);
+
+        leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
+
+        AddServerReply addServerReply = testKit.expectMsgClass(JavaTestKit.duration("5 seconds"), AddServerReply.class);
+        assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
+        assertEquals("getLeaderHint", LEADER_ID, addServerReply.getLeaderHint());
+
+        // Verify ServerConfigurationPayload entry in leader's log
+
+        verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID),
+                votingServer(NEW_SERVER_ID));
+    }
+
     @Test
     public void testAddServerForwardedToLeader() {
         DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
@@ -610,6 +622,21 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         expectFirstMatching(leaderActor, AddServer.class);
     }
 
+    @Test
+    public void testOnApplyState() {
+        RaftActorServerConfigurationSupport support = new RaftActorServerConfigurationSupport(new MockRaftActorContext());
+
+        ReplicatedLogEntry serverConfigEntry = new MockRaftActorContext.MockReplicatedLogEntry(1, 1,
+                new ServerConfigurationPayload(Collections.<ServerInfo>emptyList()));
+        boolean handled = support.handleMessage(new ApplyState(null, null, serverConfigEntry), null, ActorRef.noSender());
+        assertEquals("Message handled", true, handled);
+
+        ReplicatedLogEntry nonServerConfigEntry = new MockRaftActorContext.MockReplicatedLogEntry(1, 1,
+                new MockRaftActorContext.MockPayload("1"));
+        handled = support.handleMessage(new ApplyState(null, null, nonServerConfigEntry), null, ActorRef.noSender());
+        assertEquals("Message handled", false, handled);
+    }
+
     private ServerInfo votingServer(String id) {
         return new ServerInfo(id, true);
     }
@@ -618,6 +645,14 @@ public class RaftActorServerConfigurationSupportTest extends AbstractActorTest {
         return new ServerInfo(id, false);
     }
 
+    private TestActorRef<MessageCollectorActor> newLeaderCollectorActor(MockLeaderRaftActor leaderRaftActor) {
+        TestActorRef<MessageCollectorActor> leaderCollectorActor = actorFactory.createTestActor(
+                MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
+                actorFactory.generateActorId(LEADER_ID + "Collector"));
+        leaderRaftActor.setCollectorActor(leaderCollectorActor);
+        return leaderCollectorActor;
+    }
+
     private static void verifyServerConfigurationPayloadEntry(ReplicatedLog log, ServerInfo... expected) {
         ReplicatedLogEntry logEntry = log.get(log.lastIndex());
         assertEquals("Last log entry payload class", ServerConfigurationPayload.class, logEntry.getData().getClass());