opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/NonVotingFollowerIntegrationTest.java

   1 /*
   2  * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft;
   9
  10 import static org.junit.Assert.assertEquals;
  11
  12 import akka.actor.ActorRef;
  13 import akka.dispatch.Dispatchers;
  14 import com.google.common.base.Optional;
  15 import com.google.common.collect.ImmutableMap;
  16 import com.google.common.collect.Sets;
  17 import java.util.Arrays;
  18 import java.util.concurrent.TimeUnit;
  19 import org.junit.Test;
  20 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
  21 import org.opendaylight.controller.cluster.raft.AbstractRaftActorIntegrationTest.TestRaftActor.Builder;
  22 import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
  23 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
  24 import org.opendaylight.controller.cluster.raft.base.messages.SnapshotComplete;
  25 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
  26 import org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload;
  27 import org.opendaylight.controller.cluster.raft.persisted.ServerInfo;
  28 import org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry;
  29 import org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm;
  30 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
  31 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
  32 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
  33 import scala.concurrent.duration.FiniteDuration;
  34
  35 /**
  36  * Integration test for various scenarios involving non-voting followers.
  37  *
  38  * @author Thomas Pantelis
  39  */
  40 public class NonVotingFollowerIntegrationTest extends AbstractRaftActorIntegrationTest {
  41     private TestRaftActor followerInstance;
  42     private TestRaftActor leaderInstance;
  43     private final Builder follower1Builder = TestRaftActor.newBuilder();
  44
  45     /**
  46      * Tests non-voting follower re-sync after the non-persistent leader restarts with an empty log. In this
  47      * case the follower's log will be ahead of the leader's log as the follower retains the previous
  48      * data in memory. The leader must force an install snapshot to re-sync the follower's state.
  49      */
  50     @Test
  51     public void testFollowerResyncWithEmptyLeaderLogAfterNonPersistentLeaderRestart() {
  52         testLog.info("testFollowerResyncWithEmptyLeaderLogAfterNonPersistentLeaderRestart starting");
  53
  54         setupLeaderAndNonVotingFollower();
  55
  56         // Add log entries and verify they are committed and applied by both nodes.
  57
  58         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
  59         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
  60         expSnapshotState.add(sendPayloadData(leaderActor, "two"));
  61
  62         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
  63         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
  64
  65         assertEquals("Leader journal lastIndex", 2, leaderContext.getReplicatedLog().lastIndex());
  66         assertEquals("Leader commit index", 2, leaderContext.getCommitIndex());
  67         assertEquals("Follower journal lastIndex", 2, follower1Context.getReplicatedLog().lastIndex());
  68         assertEquals("Follower commit index", 2, follower1Context.getCommitIndex());
  69         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
  70
  71         // Persisted journal should only contain the ServerConfigurationPayload and 2 UpdateElectionTerm entries.
  72         assertEquals("Leader persisted journal size", 3, InMemoryJournal.get(leaderId).size());
  73
  74         // Restart the leader
  75
  76         killActor(leaderActor);
  77         MessageCollectorActor.clearMessages(follower1CollectorActor);
  78
  79         createNewLeaderActor();
  80
  81         //follower1Actor.underlyingActor().startDropMessages(AppendEntries.class);
  82
  83         currentTerm++;
  84         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
  85         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
  86         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
  87
  88         // After restart, the leader's log and the follower's log will be ahead so the leader should force an
  89         // install snapshot to re-sync the follower's log and state.
  90
  91         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
  92
  93         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
  94         assertEquals("Follower journal lastIndex", -1, follower1Context.getReplicatedLog().lastIndex());
  95         assertEquals("Follower commit index", -1, follower1Context.getCommitIndex());
  96
  97         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
  98
  99         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
 100
 101         assertEquals("Follower journal lastIndex", 0, follower1Context.getReplicatedLog().lastIndex());
 102         assertEquals("Follower commit index", 0, follower1Context.getCommitIndex());
 103         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 104
 105         testLog.info("testFollowerResyncWithEmptyLeaderLogAfterNonPersistentLeaderRestart ending");
 106     }
 107
 108     /**
 109      * Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
 110      * entries prior to re-connecting to the follower. The leader's last index will still be less than the
 111      * follower's last index corresponding to the previous data retained in memory. So the follower's log
 112      * will be ahead of the leader's log and the leader must force an install snapshot to re-sync the
 113      * follower's state.
 114      */
 115     @Test
 116     public void testFollowerResyncWithLessLeaderLogEntriesAfterNonPersistentLeaderRestart() {
 117         testLog.info("testFollowerResyncWithLessLeaderLogEntriesAfterNonPersistentLeaderRestart starting");
 118
 119         setupLeaderAndNonVotingFollower();
 120
 121         // Add log entries and verify they are committed and applied by both nodes.
 122
 123         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
 124         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
 125         expSnapshotState.add(sendPayloadData(leaderActor, "two"));
 126
 127         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
 128         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
 129
 130         assertEquals("Leader journal lastIndex", 2, leaderContext.getReplicatedLog().lastIndex());
 131         assertEquals("Leader commit index", 2, leaderContext.getCommitIndex());
 132         assertEquals("Follower journal lastIndex", 2, follower1Context.getReplicatedLog().lastIndex());
 133         assertEquals("Follower commit index", 2, follower1Context.getCommitIndex());
 134         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 135
 136         // Restart the leader
 137
 138         killActor(leaderActor);
 139         MessageCollectorActor.clearMessages(follower1CollectorActor);
 140
 141         // Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
 142         followerInstance.startDropMessages(AppendEntries.class);
 143
 144         createNewLeaderActor();
 145
 146         currentTerm++;
 147         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 148         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
 149         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
 150
 151         // Add new log entries to the leader - one less than the prior log entries
 152
 153         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
 154         expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
 155
 156         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 2);
 157         assertEquals("Leader journal lastIndex", 1, leaderContext.getReplicatedLog().lastIndex());
 158         assertEquals("Leader commit index", 1, leaderContext.getCommitIndex());
 159
 160         // Re-enable AppendEntries to the follower. The leaders previous index will be present in the
 161         // follower's but the terms won't match and the follower's log will be ahead of the leader's log
 162         // The leader should force an install snapshot to re-sync the entire follower's log and state.
 163
 164         followerInstance.stopDropMessages(AppendEntries.class);
 165         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
 166
 167         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 168         assertEquals("Follower journal lastIndex", 1, follower1Context.getReplicatedLog().lastIndex());
 169         assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
 170         assertEquals("Follower commit index", 1, follower1Context.getCommitIndex());
 171         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 172
 173         testLog.info("testFollowerResyncWithLessLeaderLogEntriesAfterNonPersistentLeaderRestart ending");
 174     }
 175
 176     /**
 177      * Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
 178      * entries prior to re-connecting to the follower. The leader's last index will be 1 greater than the
 179      * follower's last index corresponding to the previous data retained in memory. So the follower's log
 180      * will be behind the leader's log but the leader's log entries will have a higher term. In this case the
 181      * leader should force an install snapshot to re-sync the follower's state.
 182      */
 183     @Test
 184     public void testFollowerResyncWithOneMoreLeaderLogEntryAfterNonPersistentLeaderRestart() {
 185         testLog.info("testFollowerResyncWithOneMoreLeaderLogEntryAfterNonPersistentLeaderRestart starting");
 186
 187         setupLeaderAndNonVotingFollower();
 188
 189         // Add log entries and verify they are committed and applied by both nodes.
 190
 191         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
 192         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
 193
 194         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 2);
 195         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 2);
 196
 197         assertEquals("Leader journal lastIndex", 1, leaderContext.getReplicatedLog().lastIndex());
 198         assertEquals("Leader commit index", 1, leaderContext.getCommitIndex());
 199         assertEquals("Follower journal lastIndex", 1, follower1Context.getReplicatedLog().lastIndex());
 200         assertEquals("Follower commit index", 1, follower1Context.getCommitIndex());
 201         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 202
 203         // Restart the leader
 204
 205         killActor(leaderActor);
 206         MessageCollectorActor.clearMessages(follower1CollectorActor);
 207
 208         // Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
 209         followerInstance.startDropMessages(AppendEntries.class);
 210
 211         createNewLeaderActor();
 212
 213         currentTerm++;
 214         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 215         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
 216         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
 217
 218         // Add new log entries to the leader - one more than the prior log entries
 219
 220         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
 221         expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
 222         expSnapshotState.add(sendPayloadData(leaderActor, "two-1"));
 223
 224         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
 225         assertEquals("Leader journal lastIndex", 2, leaderContext.getReplicatedLog().lastIndex());
 226         assertEquals("Leader commit index", 2, leaderContext.getCommitIndex());
 227         assertEquals("Leader replicatedToAllIndex", -1, leaderInstance.getCurrentBehavior().getReplicatedToAllIndex());
 228
 229         // Re-enable AppendEntries to the follower. The follower's log will be out of sync and it should
 230         // should force the leader to install snapshot to re-sync the entire follower's log and state.
 231
 232         followerInstance.stopDropMessages(AppendEntries.class);
 233         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
 234
 235         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 236         assertEquals("Follower journal lastIndex", 2, follower1Context.getReplicatedLog().lastIndex());
 237         assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
 238         assertEquals("Follower commit index", 2, follower1Context.getCommitIndex());
 239         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 240
 241         testLog.info("testFollowerResyncWithOneMoreLeaderLogEntryAfterNonPersistentLeaderRestart ending");
 242     }
 243
 244     /**
 245      * Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
 246      * entries prior to re-connecting to the follower. The leader's last index will be greater than the
 247      * follower's last index corresponding to the previous data retained in memory. So the follower's log
 248      * will be behind the leader's log but the leader's log entries will have a higher term. It also adds a
 249      * "down" peer on restart so the leader doesn't trim its log as it's trying to resync the follower.
 250      * Eventually the follower should force the leader to install snapshot to re-sync its state.
 251      */
 252     @Test
 253     public void testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart() {
 254         testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart starting");
 255
 256         setupLeaderAndNonVotingFollower();
 257
 258         // Add log entries and verify they are committed and applied by both nodes.
 259
 260         expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
 261         expSnapshotState.add(sendPayloadData(leaderActor, "one"));
 262         expSnapshotState.add(sendPayloadData(leaderActor, "two"));
 263
 264         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
 265         MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, expSnapshotState.size());
 266
 267         long lastIndex = 2;
 268         assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
 269         assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
 270         assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
 271         assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
 272         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 273
 274         MessageCollectorActor.clearMessages(follower1CollectorActor);
 275         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
 276         assertEquals("Follower snapshot index", lastIndex - 1, follower1Context.getReplicatedLog().getSnapshotIndex());
 277         assertEquals("Follower journal size", 1, leaderContext.getReplicatedLog().size());
 278
 279         // Restart the leader
 280
 281         killActor(leaderActor);
 282         MessageCollectorActor.clearMessages(follower1CollectorActor);
 283
 284         // Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
 285         followerInstance.startDropMessages(AppendEntries.class);
 286
 287         // Add a "down" peer so the leader doesn't trim its log as it's trying to resync the follower. The
 288         // leader will keep decrementing the follower's nextIndex to try to find a matching index. Since
 289         // there is no matching index it will eventually hit index 0 which should cause the follower to
 290         // force an install snapshot upon failure to remove the conflicting indexes due to indexes 0 and 1
 291         // being in the prior snapshot and not the log.
 292         //
 293         // We also add another voting follower actor into the mix even though it shoildn't affect the
 294         // outcome.
 295         ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(
 296                 new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false),
 297                 new ServerInfo(follower2Id, true), new ServerInfo("downPeer", false)));
 298         SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, currentTerm,
 299                 persistedServerConfig);
 300
 301         InMemoryJournal.clear();
 302         InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(currentTerm, leaderId));
 303         InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
 304         InMemoryJournal.addEntry(follower2Id, 1, persistedServerConfigEntry);
 305
 306         DefaultConfigParamsImpl follower2ConfigParams = newFollowerConfigParams();
 307         follower2ConfigParams.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
 308         follower2Actor = newTestRaftActor(follower2Id, TestRaftActor.newBuilder().peerAddresses(
 309                 ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, follower1Actor.path().toString()))
 310                     .config(follower2ConfigParams).persistent(Optional.of(false)));
 311         TestRaftActor follower2Instance = follower2Actor.underlyingActor();
 312         follower2Instance.waitForRecoveryComplete();
 313         follower2CollectorActor = follower2Instance.collectorActor();
 314
 315         peerAddresses = ImmutableMap.of(follower1Id, follower1Actor.path().toString(),
 316                 follower2Id, follower2Actor.path().toString());
 317
 318         createNewLeaderActor();
 319
 320         currentTerm++;
 321         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 322         assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
 323         assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
 324
 325         // Add new log entries to the leader - several more than the prior log entries
 326
 327         expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
 328         expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
 329         expSnapshotState.add(sendPayloadData(leaderActor, "two-1"));
 330         expSnapshotState.add(sendPayloadData(leaderActor, "three-1"));
 331         expSnapshotState.add(sendPayloadData(leaderActor, "four-1"));
 332
 333         MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
 334         MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, expSnapshotState.size());
 335
 336         lastIndex = 4;
 337         assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
 338         assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
 339         assertEquals("Leader snapshot index", -1, leaderContext.getReplicatedLog().getSnapshotIndex());
 340         assertEquals("Leader replicatedToAllIndex", -1, leaderInstance.getCurrentBehavior().getReplicatedToAllIndex());
 341
 342         // Re-enable AppendEntries to the follower. The follower's log will be out of sync and it should
 343         // should force the leader to install snapshot to re-sync the entire follower's log and state.
 344
 345         followerInstance.stopDropMessages(AppendEntries.class);
 346         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
 347
 348         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 349         assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
 350         assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
 351         assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
 352         assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
 353
 354         testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart ending");
 355     }
 356
 357     @Test
 358     public void testFollowerLeaderStateChanges() {
 359         testLog.info("testFollowerLeaderStateChanges");
 360
 361         ActorRef roleChangeNotifier = factory.<MessageCollectorActor>createTestActor(
 362                 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()),
 363                 factory.generateActorId("roleChangeNotifier"));
 364         follower1Builder.roleChangeNotifier(roleChangeNotifier);
 365
 366         setupLeaderAndNonVotingFollower();
 367
 368         ((DefaultConfigParamsImpl)follower1Context.getConfigParams()).setElectionTimeoutFactor(2);
 369         ((DefaultConfigParamsImpl)follower1Context.getConfigParams())
 370                 .setHeartBeatInterval(FiniteDuration.apply(100, TimeUnit.MILLISECONDS));
 371
 372         MessageCollectorActor.clearMessages(roleChangeNotifier);
 373         follower1Actor.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
 374         followerInstance.startDropMessages(AppendEntries.class);
 375
 376         LeaderStateChanged leaderStateChanged = MessageCollectorActor.expectFirstMatching(roleChangeNotifier,
 377                 LeaderStateChanged.class);
 378         assertEquals("getLeaderId", null, leaderStateChanged.getLeaderId());
 379
 380         MessageCollectorActor.clearMessages(roleChangeNotifier);
 381         followerInstance.stopDropMessages(AppendEntries.class);
 382
 383         leaderStateChanged = MessageCollectorActor.expectFirstMatching(roleChangeNotifier,
 384                 LeaderStateChanged.class);
 385         assertEquals("getLeaderId", leaderId, leaderStateChanged.getLeaderId());
 386     }
 387
 388     private void createNewLeaderActor() {
 389         expSnapshotState.clear();
 390         leaderActor = newTestRaftActor(leaderId, TestRaftActor.newBuilder().peerAddresses(peerAddresses)
 391                 .config(leaderConfigParams).persistent(Optional.of(false)));
 392         leaderInstance = leaderActor.underlyingActor();
 393         leaderCollectorActor = leaderInstance.collectorActor();
 394         waitUntilLeader(leaderActor);
 395         leaderContext = leaderInstance.getRaftActorContext();
 396     }
 397
 398     private void setupLeaderAndNonVotingFollower() {
 399         snapshotBatchCount = 100;
 400         int persistedTerm = 1;
 401
 402         // Set up a persisted ServerConfigurationPayload with the leader voting and the follower non-voting.
 403
 404         ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(
 405                 new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false)));
 406         SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, persistedTerm,
 407                 persistedServerConfig);
 408
 409         InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(persistedTerm, leaderId));
 410         InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
 411         InMemoryJournal.addEntry(follower1Id, 1, new UpdateElectionTerm(persistedTerm, leaderId));
 412         InMemoryJournal.addEntry(follower1Id, 2, persistedServerConfigEntry);
 413
 414         DefaultConfigParamsImpl followerConfigParams = newFollowerConfigParams();
 415         follower1Actor = newTestRaftActor(follower1Id, follower1Builder.peerAddresses(
 416                 ImmutableMap.of(leaderId, testActorPath(leaderId))).config(followerConfigParams)
 417                     .persistent(Optional.of(false)));
 418
 419         peerAddresses = ImmutableMap.<String, String>builder()
 420                 .put(follower1Id, follower1Actor.path().toString()).build();
 421
 422         leaderConfigParams = newLeaderConfigParams();
 423         leaderActor = newTestRaftActor(leaderId, TestRaftActor.newBuilder().peerAddresses(peerAddresses)
 424                 .config(leaderConfigParams).persistent(Optional.of(false)));
 425
 426         followerInstance = follower1Actor.underlyingActor();
 427         follower1CollectorActor = followerInstance.collectorActor();
 428
 429         leaderInstance = leaderActor.underlyingActor();
 430         leaderCollectorActor = leaderInstance.collectorActor();
 431
 432         leaderContext = leaderInstance.getRaftActorContext();
 433         follower1Context = followerInstance.getRaftActorContext();
 434
 435         waitUntilLeader(leaderActor);
 436
 437         // Verify leader's context after startup
 438
 439         currentTerm = persistedTerm + 1;
 440         assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
 441         assertEquals("Leader server config", Sets.newHashSet(persistedServerConfig.getServerConfig()),
 442                 Sets.newHashSet(leaderContext.getPeerServerInfo(true).getServerConfig()));
 443         assertEquals("Leader isVotingMember", true, leaderContext.isVotingMember());
 444
 445         // Verify follower's context after startup
 446
 447         MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
 448         assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
 449         assertEquals("Follower server config", Sets.newHashSet(persistedServerConfig.getServerConfig()),
 450                 Sets.newHashSet(follower1Context.getPeerServerInfo(true).getServerConfig()));
 451         assertEquals("FollowerisVotingMember", false, follower1Context.isVotingMember());
 452     }
 453 }