opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorRecoverySupportTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertFalse;
  12 import static org.junit.Assert.assertTrue;
  13 import static org.mockito.ArgumentMatchers.any;
  14 import static org.mockito.ArgumentMatchers.anyInt;
  15 import static org.mockito.Mockito.doNothing;
  16 import static org.mockito.Mockito.doReturn;
  17 import static org.mockito.Mockito.never;
  18 import static org.mockito.Mockito.verify;
  19 import static org.mockito.Mockito.verifyNoMoreInteractions;
  20
  21 import akka.persistence.RecoveryCompleted;
  22 import akka.persistence.SnapshotMetadata;
  23 import akka.persistence.SnapshotOffer;
  24 import com.google.common.collect.Sets;
  25 import java.util.Arrays;
  26 import java.util.Collections;
  27 import org.junit.Before;
  28 import org.junit.Test;
  29 import org.mockito.ArgumentMatchers;
  30 import org.mockito.InOrder;
  31 import org.mockito.Mock;
  32 import org.mockito.Mockito;
  33 import org.mockito.MockitoAnnotations;
  34 import org.opendaylight.controller.cluster.DataPersistenceProvider;
  35 import org.opendaylight.controller.cluster.PersistentDataProvider;
  36 import org.opendaylight.controller.cluster.raft.MockRaftActor.MockSnapshotState;
  37 import org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload;
  38 import org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries;
  39 import org.opendaylight.controller.cluster.raft.persisted.DeleteEntries;
  40 import org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload;
  41 import org.opendaylight.controller.cluster.raft.persisted.ServerInfo;
  42 import org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry;
  43 import org.opendaylight.controller.cluster.raft.persisted.Snapshot;
  44 import org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm;
  45 import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48
  49 /**
  50  * Unit tests for RaftActorRecoverySupport.
  51  *
  52  * @author Thomas Pantelis
  53  */
  54 public class RaftActorRecoverySupportTest {
  55
  56     private static final Logger LOG = LoggerFactory.getLogger(RaftActorRecoverySupportTest.class);
  57
  58     @Mock
  59     private DataPersistenceProvider mockPersistence;
  60
  61
  62     @Mock
  63     private RaftActorRecoveryCohort mockCohort;
  64
  65     @Mock
  66     private RaftActorSnapshotCohort mockSnapshotCohort;
  67
  68     @Mock
  69     PersistentDataProvider mockPersistentProvider;
  70
  71     private RaftActorRecoverySupport support;
  72
  73     private RaftActorContext context;
  74     private final DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
  75     private final String localId = "leader";
  76
  77
  78     @Before
  79     public void setup() {
  80         MockitoAnnotations.initMocks(this);
  81
  82         context = new RaftActorContextImpl(null, null, localId, new ElectionTermImpl(mockPersistentProvider, "test",
  83                 LOG), -1, -1, Collections.<String,String>emptyMap(), configParams,
  84                 mockPersistence, applyState -> { }, LOG);
  85
  86         support = new RaftActorRecoverySupport(context, mockCohort);
  87
  88         doReturn(true).when(mockPersistence).isRecoveryApplicable();
  89
  90         context.setReplicatedLog(ReplicatedLogImpl.newInstance(context));
  91     }
  92
  93     private void sendMessageToSupport(final Object message) {
  94         sendMessageToSupport(message, false);
  95     }
  96
  97     private void sendMessageToSupport(final Object message, final boolean expComplete) {
  98         boolean complete = support.handleRecoveryMessage(message, mockPersistentProvider);
  99         assertEquals("complete", expComplete, complete);
 100     }
 101
 102     @Test
 103     public void testOnReplicatedLogEntry() {
 104         ReplicatedLogEntry logEntry = new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("1", 5));
 105
 106         sendMessageToSupport(logEntry);
 107
 108         assertEquals("Journal log size", 1, context.getReplicatedLog().size());
 109         assertEquals("Journal data size", 5, context.getReplicatedLog().dataSize());
 110         assertEquals("Last index", 1, context.getReplicatedLog().lastIndex());
 111         assertEquals("Last applied", -1, context.getLastApplied());
 112         assertEquals("Commit index", -1, context.getCommitIndex());
 113         assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
 114         assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
 115     }
 116
 117     @Test
 118     public void testOnApplyJournalEntries() {
 119         configParams.setJournalRecoveryLogBatchSize(5);
 120
 121         ReplicatedLog replicatedLog = context.getReplicatedLog();
 122         replicatedLog.append(new SimpleReplicatedLogEntry(0, 1, new MockRaftActorContext.MockPayload("0")));
 123         replicatedLog.append(new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("1")));
 124         replicatedLog.append(new SimpleReplicatedLogEntry(2, 1, new MockRaftActorContext.MockPayload("2")));
 125         replicatedLog.append(new SimpleReplicatedLogEntry(3, 1, new MockRaftActorContext.MockPayload("3")));
 126         replicatedLog.append(new SimpleReplicatedLogEntry(4, 1, new MockRaftActorContext.MockPayload("4")));
 127         replicatedLog.append(new SimpleReplicatedLogEntry(5, 1, new MockRaftActorContext.MockPayload("5")));
 128
 129         sendMessageToSupport(new ApplyJournalEntries(2));
 130
 131         assertEquals("Last applied", 2, context.getLastApplied());
 132         assertEquals("Commit index", 2, context.getCommitIndex());
 133
 134         sendMessageToSupport(new ApplyJournalEntries(4));
 135
 136         assertEquals("Last applied", 4, context.getLastApplied());
 137         assertEquals("Last applied", 4, context.getLastApplied());
 138
 139         sendMessageToSupport(new ApplyJournalEntries(5));
 140
 141         assertEquals("Last index", 5, context.getReplicatedLog().lastIndex());
 142         assertEquals("Last applied", 5, context.getLastApplied());
 143         assertEquals("Commit index", 5, context.getCommitIndex());
 144         assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
 145         assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
 146
 147         InOrder inOrder = Mockito.inOrder(mockCohort);
 148         inOrder.verify(mockCohort).startLogRecoveryBatch(5);
 149
 150         for (int i = 0; i < replicatedLog.size() - 1; i++) {
 151             inOrder.verify(mockCohort).appendRecoveredLogEntry(replicatedLog.get(i).getData());
 152         }
 153
 154         inOrder.verify(mockCohort).applyCurrentLogRecoveryBatch();
 155         inOrder.verify(mockCohort).startLogRecoveryBatch(5);
 156         inOrder.verify(mockCohort).appendRecoveredLogEntry(replicatedLog.get(replicatedLog.size() - 1).getData());
 157
 158         inOrder.verifyNoMoreInteractions();
 159     }
 160
 161     @Test
 162     public void testOnSnapshotOffer() {
 163
 164         ReplicatedLog replicatedLog = context.getReplicatedLog();
 165         replicatedLog.append(new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("1")));
 166         replicatedLog.append(new SimpleReplicatedLogEntry(2, 1, new MockRaftActorContext.MockPayload("2")));
 167         replicatedLog.append(new SimpleReplicatedLogEntry(3, 1, new MockRaftActorContext.MockPayload("3")));
 168
 169         ReplicatedLogEntry unAppliedEntry1 = new SimpleReplicatedLogEntry(4, 1,
 170                 new MockRaftActorContext.MockPayload("4", 4));
 171
 172         ReplicatedLogEntry unAppliedEntry2 = new SimpleReplicatedLogEntry(5, 1,
 173                 new MockRaftActorContext.MockPayload("5", 5));
 174
 175         long lastAppliedDuringSnapshotCapture = 3;
 176         long lastIndexDuringSnapshotCapture = 5;
 177         long electionTerm = 2;
 178         String electionVotedFor = "member-2";
 179
 180         MockSnapshotState snapshotState = new MockSnapshotState(Arrays.asList(new MockPayload("1")));
 181         Snapshot snapshot = Snapshot.create(snapshotState,
 182                 Arrays.asList(unAppliedEntry1, unAppliedEntry2), lastIndexDuringSnapshotCapture, 1,
 183                 lastAppliedDuringSnapshotCapture, 1, electionTerm, electionVotedFor, null);
 184
 185         SnapshotMetadata metadata = new SnapshotMetadata("test", 6, 12345);
 186         SnapshotOffer snapshotOffer = new SnapshotOffer(metadata , snapshot);
 187
 188         sendMessageToSupport(snapshotOffer);
 189
 190         assertEquals("Journal log size", 2, context.getReplicatedLog().size());
 191         assertEquals("Journal data size", 9, context.getReplicatedLog().dataSize());
 192         assertEquals("Last index", lastIndexDuringSnapshotCapture, context.getReplicatedLog().lastIndex());
 193         assertEquals("Last applied", lastAppliedDuringSnapshotCapture, context.getLastApplied());
 194         assertEquals("Commit index", lastAppliedDuringSnapshotCapture, context.getCommitIndex());
 195         assertEquals("Snapshot term", 1, context.getReplicatedLog().getSnapshotTerm());
 196         assertEquals("Snapshot index", lastAppliedDuringSnapshotCapture, context.getReplicatedLog().getSnapshotIndex());
 197         assertEquals("Election term", electionTerm, context.getTermInformation().getCurrentTerm());
 198         assertEquals("Election votedFor", electionVotedFor, context.getTermInformation().getVotedFor());
 199         assertFalse("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 200
 201         verify(mockCohort).applyRecoverySnapshot(snapshotState);
 202     }
 203
 204     @Test
 205     public void testOnRecoveryCompletedWithRemainingBatch() {
 206         ReplicatedLog replicatedLog = context.getReplicatedLog();
 207         replicatedLog.append(new SimpleReplicatedLogEntry(0, 1, new MockRaftActorContext.MockPayload("0")));
 208         replicatedLog.append(new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("1")));
 209
 210         sendMessageToSupport(new ApplyJournalEntries(1));
 211
 212         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 213
 214         assertEquals("Last applied", 1, context.getLastApplied());
 215         assertEquals("Commit index", 1, context.getCommitIndex());
 216
 217         InOrder inOrder = Mockito.inOrder(mockCohort);
 218         inOrder.verify(mockCohort).startLogRecoveryBatch(anyInt());
 219
 220         for (int i = 0; i < replicatedLog.size(); i++) {
 221             inOrder.verify(mockCohort).appendRecoveredLogEntry(replicatedLog.get(i).getData());
 222         }
 223
 224         inOrder.verify(mockCohort).applyCurrentLogRecoveryBatch();
 225         inOrder.verify(mockCohort).getRestoreFromSnapshot();
 226         inOrder.verifyNoMoreInteractions();
 227     }
 228
 229     @Test
 230     public void testOnRecoveryCompletedWithNoRemainingBatch() {
 231         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 232
 233         verify(mockCohort).getRestoreFromSnapshot();
 234         verifyNoMoreInteractions(mockCohort);
 235     }
 236
 237     @Test
 238     public void testOnDeleteEntries() {
 239         ReplicatedLog replicatedLog = context.getReplicatedLog();
 240         replicatedLog.append(new SimpleReplicatedLogEntry(0, 1, new MockRaftActorContext.MockPayload("0")));
 241         replicatedLog.append(new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("1")));
 242         replicatedLog.append(new SimpleReplicatedLogEntry(2, 1, new MockRaftActorContext.MockPayload("2")));
 243
 244         sendMessageToSupport(new DeleteEntries(1));
 245
 246         assertEquals("Journal log size", 1, context.getReplicatedLog().size());
 247         assertEquals("Last index", 0, context.getReplicatedLog().lastIndex());
 248     }
 249
 250     @Test
 251     public void testUpdateElectionTerm() {
 252
 253         sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
 254
 255         assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
 256         assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
 257     }
 258
 259     @Test
 260     public void testDataRecoveredWithPersistenceDisabled() {
 261         doNothing().when(mockCohort).applyRecoverySnapshot(any());
 262         doReturn(false).when(mockPersistence).isRecoveryApplicable();
 263         doReturn(10L).when(mockPersistentProvider).getLastSequenceNumber();
 264
 265         Snapshot snapshot = Snapshot.create(new MockSnapshotState(Arrays.asList(new MockPayload("1"))),
 266                 Collections.<ReplicatedLogEntry>emptyList(), 3, 1, 3, 1, -1, null, null);
 267         SnapshotOffer snapshotOffer = new SnapshotOffer(new SnapshotMetadata("test", 6, 12345), snapshot);
 268
 269         sendMessageToSupport(snapshotOffer);
 270
 271         sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
 272
 273         sendMessageToSupport(new SimpleReplicatedLogEntry(4, 1, new MockRaftActorContext.MockPayload("4")));
 274         sendMessageToSupport(new SimpleReplicatedLogEntry(5, 1, new MockRaftActorContext.MockPayload("5")));
 275
 276         sendMessageToSupport(new ApplyJournalEntries(4));
 277
 278         sendMessageToSupport(new DeleteEntries(5));
 279
 280         assertEquals("Journal log size", 0, context.getReplicatedLog().size());
 281         assertEquals("Last index", -1, context.getReplicatedLog().lastIndex());
 282         assertEquals("Last applied", -1, context.getLastApplied());
 283         assertEquals("Commit index", -1, context.getCommitIndex());
 284         assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
 285         assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
 286
 287         assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
 288         assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
 289
 290         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 291
 292         verify(mockCohort, never()).applyRecoverySnapshot(any());
 293         verify(mockCohort, never()).getRestoreFromSnapshot();
 294         verifyNoMoreInteractions(mockCohort);
 295
 296         verify(mockPersistentProvider).deleteMessages(10L);
 297     }
 298
 299     static UpdateElectionTerm updateElectionTerm(final long term, final String votedFor) {
 300         return ArgumentMatchers.argThat(
 301             other -> term == other.getCurrentTerm() && votedFor.equals(other.getVotedFor()));
 302     }
 303
 304     @Test
 305     public void testNoDataRecoveredWithPersistenceDisabled() {
 306         doReturn(false).when(mockPersistence).isRecoveryApplicable();
 307
 308         sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
 309
 310         assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
 311         assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
 312
 313         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 314
 315         verify(mockCohort).getRestoreFromSnapshot();
 316         verifyNoMoreInteractions(mockCohort, mockPersistentProvider);
 317     }
 318
 319     @Test
 320     public void testServerConfigurationPayloadApplied() {
 321         String follower1 = "follower1";
 322         String follower2 = "follower2";
 323         String follower3 = "follower3";
 324
 325         context.addToPeers(follower1, null, VotingState.VOTING);
 326         context.addToPeers(follower2, null, VotingState.VOTING);
 327
 328         //add new Server
 329         ServerConfigurationPayload obj = new ServerConfigurationPayload(Arrays.asList(
 330                 new ServerInfo(localId, true),
 331                 new ServerInfo(follower1, true),
 332                 new ServerInfo(follower2, false),
 333                 new ServerInfo(follower3, true)));
 334
 335         sendMessageToSupport(new SimpleReplicatedLogEntry(0, 1, obj));
 336
 337         //verify new peers
 338         assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 339         assertEquals("New peer Ids", Sets.newHashSet(follower1, follower2, follower3),
 340                 Sets.newHashSet(context.getPeerIds()));
 341         assertEquals("follower1 isVoting", true, context.getPeerInfo(follower1).isVoting());
 342         assertEquals("follower2 isVoting", false, context.getPeerInfo(follower2).isVoting());
 343         assertEquals("follower3 isVoting", true, context.getPeerInfo(follower3).isVoting());
 344
 345         sendMessageToSupport(new ApplyJournalEntries(0));
 346
 347         verify(mockCohort, never()).startLogRecoveryBatch(anyInt());
 348         verify(mockCohort, never()).appendRecoveredLogEntry(any(Payload.class));
 349
 350         //remove existing follower1
 351         obj = new ServerConfigurationPayload(Arrays.asList(
 352                 new ServerInfo(localId, true),
 353                 new ServerInfo("follower2", true),
 354                 new ServerInfo("follower3", true)));
 355
 356         sendMessageToSupport(new SimpleReplicatedLogEntry(1, 1, obj));
 357
 358         //verify new peers
 359         assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 360         assertEquals("New peer Ids", Sets.newHashSet(follower2, follower3), Sets.newHashSet(context.getPeerIds()));
 361     }
 362
 363     @Test
 364     public void testServerConfigurationPayloadAppliedWithPersistenceDisabled() {
 365         doReturn(false).when(mockPersistence).isRecoveryApplicable();
 366
 367         String follower = "follower";
 368         ServerConfigurationPayload obj = new ServerConfigurationPayload(Arrays.asList(
 369                 new ServerInfo(localId, true), new ServerInfo(follower, true)));
 370
 371         sendMessageToSupport(new SimpleReplicatedLogEntry(0, 1, obj));
 372
 373         //verify new peers
 374         assertEquals("New peer Ids", Sets.newHashSet(follower), Sets.newHashSet(context.getPeerIds()));
 375     }
 376
 377     @Test
 378     public void testOnSnapshotOfferWithServerConfiguration() {
 379         long electionTerm = 2;
 380         String electionVotedFor = "member-2";
 381         ServerConfigurationPayload serverPayload = new ServerConfigurationPayload(Arrays.asList(
 382                                                         new ServerInfo(localId, true),
 383                                                         new ServerInfo("follower1", true),
 384                                                         new ServerInfo("follower2", true)));
 385
 386         MockSnapshotState snapshotState = new MockSnapshotState(Arrays.asList(new MockPayload("1")));
 387         Snapshot snapshot = Snapshot.create(snapshotState, Collections.<ReplicatedLogEntry>emptyList(),
 388                 -1, -1, -1, -1, electionTerm, electionVotedFor, serverPayload);
 389
 390         SnapshotMetadata metadata = new SnapshotMetadata("test", 6, 12345);
 391         SnapshotOffer snapshotOffer = new SnapshotOffer(metadata , snapshot);
 392
 393         sendMessageToSupport(snapshotOffer);
 394
 395         assertEquals("Journal log size", 0, context.getReplicatedLog().size());
 396         assertEquals("Election term", electionTerm, context.getTermInformation().getCurrentTerm());
 397         assertEquals("Election votedFor", electionVotedFor, context.getTermInformation().getVotedFor());
 398         assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 399         assertEquals("Peer List", Sets.newHashSet("follower1", "follower2"),
 400             Sets.newHashSet(context.getPeerIds()));
 401     }
 402 }