opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/NonVotingFollowerIntegrationTest.java

   1 /*
   2  * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import akka.actor.ActorRef;
  12 import akka.dispatch.Dispatchers;
  13 import com.google.common.base.Optional;
  14 import com.google.common.collect.ImmutableMap;
  15 import com.google.common.collect.Sets;
  16 import java.util.Arrays;
  17 import java.util.concurrent.TimeUnit;
  18 import org.junit.Test;
  19 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
  20 import org.opendaylight.controller.cluster.raft.AbstractRaftActorIntegrationTest.TestRaftActor.Builder;
  21 import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
  22 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
  23 import org.opendaylight.controller.cluster.raft.base.messages.SnapshotComplete;
  24 import org.opendaylight.controller.cluster.raft.base.messages.UpdateElectionTerm;
  25 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
  26 import org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload;
  27 import org.opendaylight.controller.cluster.raft.persisted.ServerInfo;
  28 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
  29 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
  30 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
  31 import scala.concurrent.duration.FiniteDuration;
  32
  33 /**
  34  * Integration test for various scenarios involving non-voting followers.
  35  *
  36  * @author Thomas Pantelis
  37  */
  38 public class NonVotingFollowerIntegrationTest extends AbstractRaftActorIntegrationTest {
  39     private TestRaftActor followerInstance;
  40     private TestRaftActor leaderInstance;
  41     private final Builder follower1Builder = TestRaftActor.newBuilder();
  42
  43     /**
  44      * Tests non-voting follower re-sync after the non-persistent leader restarts with an empty log. In this
  45      * case the follower's log will be ahead of the leader's log as the follower retains the previous
  46      * data in memory. The leader must force an install snapshot to re-sync the follower's state.
  47      */
  48     @Test
  49     public void testFollowerResyncWithEmptyLeaderLogAfterNonPersistentLeaderRestart() {
  50         testLog.info("testFollowerResyncWithEmptyLeaderLogAfterNonPersistentLeaderRestart starting");
  51
  52         setupLeaderAndNonVotingFollower();
  53
  54         // Add log entries and verify they are committed and applied by both nodes.
  55
  56         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
  57         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
  58         expSnapshotState.add(sendPayloadData(leaderActor, "two"));
  59
  60         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
  61         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
  62
  63         assertEquals("Leader journal lastIndex", 2, leaderContext.getReplicatedLog().lastIndex());
  64         assertEquals("Leader commit index", 2, leaderContext.getCommitIndex());
  65         assertEquals("Follower journal lastIndex", 2, follower1Context.getReplicatedLog().lastIndex());
  66         assertEquals("Follower commit index", 2, follower1Context.getCommitIndex());
  67         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
  68
  69         // Persisted journal should only contain the ServerConfigurationPayload and 2 UpdateElectionTerm entries.
  70         assertEquals("Leader persisted journal size", 3, InMemoryJournal.get(leaderId).size());
  71
  72         // Restart the leader
  73
  74         killActor(leaderActor);
  75         MessageCollectorActor.clearMessages(follower1CollectorActor);
  76
  77         createNewLeaderActor();
  78
  79         //follower1Actor.underlyingActor().startDropMessages(AppendEntries.class);
  80
  81         currentTerm++;
  82         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
  83         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
  84         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
  85
  86         // After restart, the leader's log and the follower's log will be ahead so the leader should force an
  87         // install snapshot to re-sync the follower's log and state.
  88
  89         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
  90
  91         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
  92         assertEquals("Follower journal lastIndex", -1, follower1Context.getReplicatedLog().lastIndex());
  93         assertEquals("Follower commit index", -1, follower1Context.getCommitIndex());
  94
  95         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
  96
  97         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
  98
  99         assertEquals("Follower journal lastIndex", 0, follower1Context.getReplicatedLog().lastIndex());
 100         assertEquals("Follower commit index", 0, follower1Context.getCommitIndex());
 101         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 102
 103         testLog.info("testFollowerResyncWithEmptyLeaderLogAfterNonPersistentLeaderRestart ending");
 104     }
 105
 106     /**
 107      * Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
 108      * entries prior to re-connecting to the follower. The leader's last index will still be less than the
 109      * follower's last index corresponding to the previous data retained in memory. So the follower's log
 110      * will be ahead of the leader's log and the leader must force an install snapshot to re-sync the
 111      * follower's state.
 112      */
 113     @Test
 114     public void testFollowerResyncWithLessLeaderLogEntriesAfterNonPersistentLeaderRestart() {
 115         testLog.info("testFollowerResyncWithLessLeaderLogEntriesAfterNonPersistentLeaderRestart starting");
 116
 117         setupLeaderAndNonVotingFollower();
 118
 119         // Add log entries and verify they are committed and applied by both nodes.
 120
 121         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
 122         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
 123         expSnapshotState.add(sendPayloadData(leaderActor, "two"));
 124
 125         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
 126         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
 127
 128         assertEquals("Leader journal lastIndex", 2, leaderContext.getReplicatedLog().lastIndex());
 129         assertEquals("Leader commit index", 2, leaderContext.getCommitIndex());
 130         assertEquals("Follower journal lastIndex", 2, follower1Context.getReplicatedLog().lastIndex());
 131         assertEquals("Follower commit index", 2, follower1Context.getCommitIndex());
 132         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 133
 134         // Restart the leader
 135
 136         killActor(leaderActor);
 137         MessageCollectorActor.clearMessages(follower1CollectorActor);
 138
 139         // Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
 140         followerInstance.startDropMessages(AppendEntries.class);
 141
 142         createNewLeaderActor();
 143
 144         currentTerm++;
 145         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 146         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
 147         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
 148
 149         // Add new log entries to the leader - one less than the prior log entries
 150
 151         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
 152         expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
 153
 154         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 2);
 155         assertEquals("Leader journal lastIndex", 1, leaderContext.getReplicatedLog().lastIndex());
 156         assertEquals("Leader commit index", 1, leaderContext.getCommitIndex());
 157
 158         // Re-enable AppendEntries to the follower. The leaders previous index will be present in the
 159         // follower's but the terms won't match and the follower's log will be ahead of the leader's log
 160         // The leader should force an install snapshot to re-sync the entire follower's log and state.
 161
 162         followerInstance.stopDropMessages(AppendEntries.class);
 163         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
 164
 165         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 166         assertEquals("Follower journal lastIndex", 1, follower1Context.getReplicatedLog().lastIndex());
 167         assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
 168         assertEquals("Follower commit index", 1, follower1Context.getCommitIndex());
 169         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 170
 171         testLog.info("testFollowerResyncWithLessLeaderLogEntriesAfterNonPersistentLeaderRestart ending");
 172     }
 173
 174     /**
 175      * Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
 176      * entries prior to re-connecting to the follower. The leader's last index will be 1 greater than the
 177      * follower's last index corresponding to the previous data retained in memory. So the follower's log
 178      * will be behind the leader's log but the leader's log entries will have a higher term. In this case the
 179      * leader should force an install snapshot to re-sync the follower's state.
 180      */
 181     @Test
 182     public void testFollowerResyncWithOneMoreLeaderLogEntryAfterNonPersistentLeaderRestart() {
 183         testLog.info("testFollowerResyncWithOneMoreLeaderLogEntryAfterNonPersistentLeaderRestart starting");
 184
 185         setupLeaderAndNonVotingFollower();
 186
 187         // Add log entries and verify they are committed and applied by both nodes.
 188
 189         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
 190         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
 191
 192         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 2);
 193         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 2);
 194
 195         assertEquals("Leader journal lastIndex", 1, leaderContext.getReplicatedLog().lastIndex());
 196         assertEquals("Leader commit index", 1, leaderContext.getCommitIndex());
 197         assertEquals("Follower journal lastIndex", 1, follower1Context.getReplicatedLog().lastIndex());
 198         assertEquals("Follower commit index", 1, follower1Context.getCommitIndex());
 199         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 200
 201         // Restart the leader
 202
 203         killActor(leaderActor);
 204         MessageCollectorActor.clearMessages(follower1CollectorActor);
 205
 206         // Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
 207         followerInstance.startDropMessages(AppendEntries.class);
 208
 209         createNewLeaderActor();
 210
 211         currentTerm++;
 212         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 213         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
 214         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
 215
 216         // Add new log entries to the leader - one more than the prior log entries
 217
 218         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
 219         expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
 220         expSnapshotState.add(sendPayloadData(leaderActor, "two-1"));
 221
 222         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
 223         assertEquals("Leader journal lastIndex", 2, leaderContext.getReplicatedLog().lastIndex());
 224         assertEquals("Leader commit index", 2, leaderContext.getCommitIndex());
 225         assertEquals("Leader replicatedToAllIndex", -1, leaderInstance.getCurrentBehavior().getReplicatedToAllIndex());
 226
 227         // Re-enable AppendEntries to the follower. The follower's log will be out of sync and it should
 228         // should force the leader to install snapshot to re-sync the entire follower's log and state.
 229
 230         followerInstance.stopDropMessages(AppendEntries.class);
 231         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
 232
 233         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 234         assertEquals("Follower journal lastIndex", 2, follower1Context.getReplicatedLog().lastIndex());
 235         assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
 236         assertEquals("Follower commit index", 2, follower1Context.getCommitIndex());
 237         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 238
 239         testLog.info("testFollowerResyncWithOneMoreLeaderLogEntryAfterNonPersistentLeaderRestart ending");
 240     }
 241
 242     /**
 243      * Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
 244      * entries prior to re-connecting to the follower. The leader's last index will be greater than the
 245      * follower's last index corresponding to the previous data retained in memory. So the follower's log
 246      * will be behind the leader's log but the leader's log entries will have a higher term. It also adds a
 247      * "down" peer on restart so the leader doesn't trim its log as it's trying to resync the follower.
 248      * Eventually the follower should force the leader to install snapshot to re-sync its state.
 249      */
 250     @Test
 251     public void testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart() {
 252         testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart starting");
 253
 254         setupLeaderAndNonVotingFollower();
 255
 256         // Add log entries and verify they are committed and applied by both nodes.
 257
 258         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
 259         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
 260         expSnapshotState.add(sendPayloadData(leaderActor, "two"));
 261
 262         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
 263         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, expSnapshotState.size());
 264
 265         long lastIndex = 2;
 266         assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
 267         assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
 268         assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
 269         assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
 270         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 271
 272         MessageCollectorActor.clearMessages(follower1CollectorActor);
 273         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
 274         assertEquals("Follower snapshot index", lastIndex - 1, follower1Context.getReplicatedLog().getSnapshotIndex());
 275         assertEquals("Follower journal size", 1, leaderContext.getReplicatedLog().size());
 276
 277         // Restart the leader
 278
 279         killActor(leaderActor);
 280         MessageCollectorActor.clearMessages(follower1CollectorActor);
 281
 282         // Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
 283         followerInstance.startDropMessages(AppendEntries.class);
 284
 285         // Add a "down" peer so the leader doesn't trim its log as it's trying to resync the follower. The
 286         // leader will keep decrementing the follower's nextIndex to try to find a matching index. Since
 287         // there is no matching index it will eventually hit index 0 which should cause the follower to
 288         // force an install snapshot upon failure to remove the conflicting indexes due to indexes 0 and 1
 289         // being in the prior snapshot and not the log.
 290         //
 291         // We also add another voting follower actor into the mix even though it shoildn't affect the
 292         // outcome.
 293         ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(
 294                 new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false),
 295                 new ServerInfo(follower2Id, true), new ServerInfo("downPeer", false)));
 296         ReplicatedLogImplEntry persistedServerConfigEntry = new ReplicatedLogImplEntry(0, currentTerm,
 297                 persistedServerConfig);
 298
 299         InMemoryJournal.clear();
 300         InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(currentTerm, leaderId));
 301         InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
 302         InMemoryJournal.addEntry(follower2Id, 1, persistedServerConfigEntry);
 303
 304         DefaultConfigParamsImpl follower2ConfigParams = newFollowerConfigParams();
 305         follower2ConfigParams.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
 306         follower2Actor = newTestRaftActor(follower2Id, TestRaftActor.newBuilder().peerAddresses(
 307                 ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, follower1Actor.path().toString())).
 308                     config(follower2ConfigParams).persistent(Optional.of(false)));
 309         TestRaftActor follower2Instance = follower2Actor.underlyingActor();
 310         follower2Instance.waitForRecoveryComplete();
 311         follower2CollectorActor = follower2Instance.collectorActor();
 312
 313         peerAddresses = ImmutableMap.of(follower1Id, follower1Actor.path().toString(),
 314                 follower2Id, follower2Actor.path().toString());
 315
 316         createNewLeaderActor();
 317
 318         currentTerm++;
 319         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 320         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
 321         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
 322
 323         // Add new log entries to the leader - several more than the prior log entries
 324
 325         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
 326         expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
 327         expSnapshotState.add(sendPayloadData(leaderActor, "two-1"));
 328         expSnapshotState.add(sendPayloadData(leaderActor, "three-1"));
 329         expSnapshotState.add(sendPayloadData(leaderActor, "four-1"));
 330
 331         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
 332         MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, expSnapshotState.size());
 333
 334         lastIndex = 4;
 335         assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
 336         assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
 337         assertEquals("Leader snapshot index", -1, leaderContext.getReplicatedLog().getSnapshotIndex());
 338         assertEquals("Leader replicatedToAllIndex", -1, leaderInstance.getCurrentBehavior().getReplicatedToAllIndex());
 339
 340         // Re-enable AppendEntries to the follower. The follower's log will be out of sync and it should
 341         // should force the leader to install snapshot to re-sync the entire follower's log and state.
 342
 343         followerInstance.stopDropMessages(AppendEntries.class);
 344         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
 345
 346         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 347         assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
 348         assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
 349         assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
 350         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 351
 352         testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart ending");
 353     }
 354
 355     @Test
 356     public void testFollowerLeaderStateChanges() {
 357         testLog.info("testFollowerLeaderStateChanges");
 358
 359         ActorRef roleChangeNotifier = factory.<MessageCollectorActor>createTestActor(
 360                 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
 361                 factory.generateActorId("roleChangeNotifier"));
 362         follower1Builder.roleChangeNotifier(roleChangeNotifier);
 363
 364         setupLeaderAndNonVotingFollower();
 365
 366         ((DefaultConfigParamsImpl)follower1Context.getConfigParams()).setElectionTimeoutFactor(2);
 367         ((DefaultConfigParamsImpl)follower1Context.getConfigParams()).
 368                 setHeartBeatInterval(FiniteDuration.apply(100, TimeUnit.MILLISECONDS));
 369
 370         MessageCollectorActor.clearMessages(roleChangeNotifier);
 371         follower1Actor.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
 372         followerInstance.startDropMessages(AppendEntries.class);
 373
 374         LeaderStateChanged leaderStateChanged = MessageCollectorActor.expectFirstMatching(roleChangeNotifier,
 375                 LeaderStateChanged.class);
 376         assertEquals("getLeaderId", null, leaderStateChanged.getLeaderId());
 377
 378         MessageCollectorActor.clearMessages(roleChangeNotifier);
 379         followerInstance.stopDropMessages(AppendEntries.class);
 380
 381         leaderStateChanged = MessageCollectorActor.expectFirstMatching(roleChangeNotifier,
 382                 LeaderStateChanged.class);
 383         assertEquals("getLeaderId", leaderId, leaderStateChanged.getLeaderId());
 384     }
 385
 386     private void createNewLeaderActor() {
 387         expSnapshotState.clear();
 388         leaderActor = newTestRaftActor(leaderId, TestRaftActor.newBuilder().peerAddresses(peerAddresses).
 389                 config(leaderConfigParams).persistent(Optional.of(false)));
 390         leaderInstance = leaderActor.underlyingActor();
 391         leaderCollectorActor = leaderInstance.collectorActor();
 392         waitUntilLeader(leaderActor);
 393         leaderContext = leaderInstance.getRaftActorContext();
 394     }
 395
 396     private void setupLeaderAndNonVotingFollower() {
 397         snapshotBatchCount = 100;
 398         int initialTerm = 1;
 399
 400         // Set up a persisted ServerConfigurationPayload with the leader voting and the follower non-voting.
 401
 402         ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(
 403                 new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false)));
 404         ReplicatedLogImplEntry persistedServerConfigEntry = new ReplicatedLogImplEntry(0, initialTerm,
 405                 persistedServerConfig);
 406
 407         InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(initialTerm, leaderId));
 408         InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
 409         InMemoryJournal.addEntry(follower1Id, 1, new UpdateElectionTerm(initialTerm, leaderId));
 410         InMemoryJournal.addEntry(follower1Id, 2, persistedServerConfigEntry);
 411
 412         DefaultConfigParamsImpl followerConfigParams = newFollowerConfigParams();
 413         follower1Actor = newTestRaftActor(follower1Id, follower1Builder.peerAddresses(
 414                 ImmutableMap.of(leaderId, testActorPath(leaderId))).config(followerConfigParams).
 415                     persistent(Optional.of(false)));
 416
 417         peerAddresses = ImmutableMap.<String, String>builder().
 418                 put(follower1Id, follower1Actor.path().toString()).build();
 419
 420         leaderConfigParams = newLeaderConfigParams();
 421         leaderActor = newTestRaftActor(leaderId, TestRaftActor.newBuilder().peerAddresses(peerAddresses).
 422                 config(leaderConfigParams).persistent(Optional.of(false)));
 423
 424         followerInstance = follower1Actor.underlyingActor();
 425         follower1CollectorActor = followerInstance.collectorActor();
 426
 427         leaderInstance = leaderActor.underlyingActor();
 428         leaderCollectorActor = leaderInstance.collectorActor();
 429
 430         leaderContext = leaderInstance.getRaftActorContext();
 431         follower1Context = followerInstance.getRaftActorContext();
 432
 433         waitUntilLeader(leaderActor);
 434
 435         // Verify leader's context after startup
 436
 437         currentTerm = initialTerm + 1;
 438         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 439         assertEquals("Leader server config", Sets.newHashSet(persistedServerConfig.getServerConfig()),
 440                 Sets.newHashSet(leaderContext.getPeerServerInfo(true).getServerConfig()));
 441         assertEquals("Leader isVotingMember", true, leaderContext.isVotingMember());
 442
 443         // Verify follower's context after startup
 444
 445         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
 446         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 447         assertEquals("Follower server config", Sets.newHashSet(persistedServerConfig.getServerConfig()),
 448                 Sets.newHashSet(follower1Context.getPeerServerInfo(true).getServerConfig()));
 449         assertEquals("FollowerisVotingMember", false, follower1Context.isVotingMember());
 450     }
 451 }