opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/RaftActorRecoverySupportTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertFalse;
  12 import static org.junit.Assert.assertTrue;
  13 import static org.mockito.AdditionalMatchers.aryEq;
  14 import static org.mockito.Matchers.any;
  15 import static org.mockito.Matchers.anyInt;
  16 import static org.mockito.Mockito.doNothing;
  17 import static org.mockito.Mockito.doReturn;
  18 import static org.mockito.Mockito.never;
  19 import static org.mockito.Mockito.verify;
  20 import static org.mockito.Mockito.verifyNoMoreInteractions;
  21 import akka.persistence.RecoveryCompleted;
  22 import akka.persistence.SnapshotMetadata;
  23 import akka.persistence.SnapshotOffer;
  24 import com.google.common.collect.Sets;
  25 import java.util.Arrays;
  26 import java.util.Collections;
  27 import org.hamcrest.Description;
  28 import org.junit.Before;
  29 import org.junit.Test;
  30 import org.mockito.ArgumentMatcher;
  31 import org.mockito.InOrder;
  32 import org.mockito.Matchers;
  33 import org.mockito.Mock;
  34 import org.mockito.Mockito;
  35 import org.mockito.MockitoAnnotations;
  36 import org.opendaylight.controller.cluster.DataPersistenceProvider;
  37 import org.opendaylight.controller.cluster.PersistentDataProvider;
  38 import org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries;
  39 import org.opendaylight.controller.cluster.raft.persisted.DeleteEntries;
  40 import org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload;
  41 import org.opendaylight.controller.cluster.raft.persisted.ServerInfo;
  42 import org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm;
  43 import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 /**
  48  * Unit tests for RaftActorRecoverySupport.
  49  *
  50  * @author Thomas Pantelis
  51  */
  52 public class RaftActorRecoverySupportTest {
  53
  54     private static final Logger LOG = LoggerFactory.getLogger(RaftActorRecoverySupportTest.class);
  55
  56     @Mock
  57     private DataPersistenceProvider mockPersistence;
  58
  59
  60     @Mock
  61     private RaftActorRecoveryCohort mockCohort;
  62
  63     @Mock
  64     PersistentDataProvider mockPersistentProvider;
  65
  66     private RaftActorRecoverySupport support;
  67
  68     private RaftActorContext context;
  69     private final DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
  70     private final String localId = "leader";
  71
  72
  73     @Before
  74     public void setup() {
  75         MockitoAnnotations.initMocks(this);
  76
  77         context = new RaftActorContextImpl(null, null, localId, new ElectionTermImpl(mockPersistentProvider, "test", LOG),
  78                 -1, -1, Collections.<String,String>emptyMap(), configParams, mockPersistence, LOG);
  79
  80         support = new RaftActorRecoverySupport(context, mockCohort);
  81
  82         doReturn(true).when(mockPersistence).isRecoveryApplicable();
  83
  84         context.setReplicatedLog(ReplicatedLogImpl.newInstance(context));
  85     }
  86
  87     private void sendMessageToSupport(Object message) {
  88         sendMessageToSupport(message, false);
  89     }
  90
  91     private void sendMessageToSupport(Object message, boolean expComplete) {
  92         boolean complete = support.handleRecoveryMessage(message, mockPersistentProvider);
  93         assertEquals("complete", expComplete, complete);
  94     }
  95
  96     @Test
  97     public void testOnReplicatedLogEntry() {
  98         MockRaftActorContext.MockReplicatedLogEntry logEntry = new MockRaftActorContext.MockReplicatedLogEntry(1,
  99                 1, new MockRaftActorContext.MockPayload("1", 5));
 100
 101         sendMessageToSupport(logEntry);
 102
 103         assertEquals("Journal log size", 1, context.getReplicatedLog().size());
 104         assertEquals("Journal data size", 5, context.getReplicatedLog().dataSize());
 105         assertEquals("Last index", 1, context.getReplicatedLog().lastIndex());
 106         assertEquals("Last applied", -1, context.getLastApplied());
 107         assertEquals("Commit index", -1, context.getCommitIndex());
 108         assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
 109         assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
 110     }
 111
 112     @Test
 113     public void testOnApplyJournalEntries() {
 114         configParams.setJournalRecoveryLogBatchSize(5);
 115
 116         ReplicatedLog replicatedLog = context.getReplicatedLog();
 117         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 118                 0, new MockRaftActorContext.MockPayload("0")));
 119         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 120                 1, new MockRaftActorContext.MockPayload("1")));
 121         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 122                 2, new MockRaftActorContext.MockPayload("2")));
 123         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 124                 3, new MockRaftActorContext.MockPayload("3")));
 125         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 126                 4, new MockRaftActorContext.MockPayload("4")));
 127         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 128                 5, new MockRaftActorContext.MockPayload("5")));
 129
 130         sendMessageToSupport(new ApplyJournalEntries(2));
 131
 132         assertEquals("Last applied", 2, context.getLastApplied());
 133         assertEquals("Commit index", 2, context.getCommitIndex());
 134
 135         sendMessageToSupport(new ApplyJournalEntries(4));
 136
 137         assertEquals("Last applied", 4, context.getLastApplied());
 138         assertEquals("Last applied", 4, context.getLastApplied());
 139
 140         sendMessageToSupport(new ApplyJournalEntries(5));
 141
 142         assertEquals("Last index", 5, context.getReplicatedLog().lastIndex());
 143         assertEquals("Last applied", 5, context.getLastApplied());
 144         assertEquals("Commit index", 5, context.getCommitIndex());
 145         assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
 146         assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
 147
 148         InOrder inOrder = Mockito.inOrder(mockCohort);
 149         inOrder.verify(mockCohort).startLogRecoveryBatch(5);
 150
 151         for(int i = 0; i < replicatedLog.size() - 1; i++) {
 152             inOrder.verify(mockCohort).appendRecoveredLogEntry(replicatedLog.get(i).getData());
 153         }
 154
 155         inOrder.verify(mockCohort).applyCurrentLogRecoveryBatch();
 156         inOrder.verify(mockCohort).startLogRecoveryBatch(5);
 157         inOrder.verify(mockCohort).appendRecoveredLogEntry(replicatedLog.get(replicatedLog.size() - 1).getData());
 158
 159         inOrder.verifyNoMoreInteractions();
 160     }
 161
 162     @Test
 163     public void testOnSnapshotOffer() {
 164
 165         ReplicatedLog replicatedLog = context.getReplicatedLog();
 166         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 167                 1, new MockRaftActorContext.MockPayload("1")));
 168         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 169                 2, new MockRaftActorContext.MockPayload("2")));
 170         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 171                 3, new MockRaftActorContext.MockPayload("3")));
 172
 173         byte[] snapshotBytes = {1,2,3,4,5};
 174
 175         ReplicatedLogEntry unAppliedEntry1 = new MockRaftActorContext.MockReplicatedLogEntry(1,
 176                 4, new MockRaftActorContext.MockPayload("4", 4));
 177
 178         ReplicatedLogEntry unAppliedEntry2 = new MockRaftActorContext.MockReplicatedLogEntry(1,
 179                 5, new MockRaftActorContext.MockPayload("5", 5));
 180
 181         long lastAppliedDuringSnapshotCapture = 3;
 182         long lastIndexDuringSnapshotCapture = 5;
 183         long electionTerm = 2;
 184         String electionVotedFor = "member-2";
 185
 186         Snapshot snapshot = Snapshot.create(snapshotBytes, Arrays.asList(unAppliedEntry1, unAppliedEntry2),
 187                 lastIndexDuringSnapshotCapture, 1, lastAppliedDuringSnapshotCapture, 1, electionTerm, electionVotedFor);
 188
 189         SnapshotMetadata metadata = new SnapshotMetadata("test", 6, 12345);
 190         SnapshotOffer snapshotOffer = new SnapshotOffer(metadata , snapshot);
 191
 192         sendMessageToSupport(snapshotOffer);
 193
 194         assertEquals("Journal log size", 2, context.getReplicatedLog().size());
 195         assertEquals("Journal data size", 9, context.getReplicatedLog().dataSize());
 196         assertEquals("Last index", lastIndexDuringSnapshotCapture, context.getReplicatedLog().lastIndex());
 197         assertEquals("Last applied", lastAppliedDuringSnapshotCapture, context.getLastApplied());
 198         assertEquals("Commit index", lastAppliedDuringSnapshotCapture, context.getCommitIndex());
 199         assertEquals("Snapshot term", 1, context.getReplicatedLog().getSnapshotTerm());
 200         assertEquals("Snapshot index", lastAppliedDuringSnapshotCapture, context.getReplicatedLog().getSnapshotIndex());
 201         assertEquals("Election term", electionTerm, context.getTermInformation().getCurrentTerm());
 202         assertEquals("Election votedFor", electionVotedFor, context.getTermInformation().getVotedFor());
 203         assertFalse("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 204
 205         verify(mockCohort).applyRecoverySnapshot(snapshotBytes);
 206     }
 207
 208     @Test
 209     public void testOnRecoveryCompletedWithRemainingBatch() {
 210         ReplicatedLog replicatedLog = context.getReplicatedLog();
 211         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 212                 0, new MockRaftActorContext.MockPayload("0")));
 213         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 214                 1, new MockRaftActorContext.MockPayload("1")));
 215
 216         sendMessageToSupport(new ApplyJournalEntries(1));
 217
 218         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 219
 220         assertEquals("Last applied", 1, context.getLastApplied());
 221         assertEquals("Commit index", 1, context.getCommitIndex());
 222
 223         InOrder inOrder = Mockito.inOrder(mockCohort);
 224         inOrder.verify(mockCohort).startLogRecoveryBatch(anyInt());
 225
 226         for(int i = 0; i < replicatedLog.size(); i++) {
 227             inOrder.verify(mockCohort).appendRecoveredLogEntry(replicatedLog.get(i).getData());
 228         }
 229
 230         inOrder.verify(mockCohort).applyCurrentLogRecoveryBatch();
 231         inOrder.verify(mockCohort).getRestoreFromSnapshot();
 232         inOrder.verifyNoMoreInteractions();
 233     }
 234
 235     @Test
 236     public void testOnRecoveryCompletedWithNoRemainingBatch() {
 237         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 238
 239         verify(mockCohort).getRestoreFromSnapshot();
 240         verifyNoMoreInteractions(mockCohort);
 241     }
 242
 243     @Test
 244     public void testOnDeleteEntries() {
 245         ReplicatedLog replicatedLog = context.getReplicatedLog();
 246         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 247                 0, new MockRaftActorContext.MockPayload("0")));
 248         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 249                 1, new MockRaftActorContext.MockPayload("1")));
 250         replicatedLog.append(new MockRaftActorContext.MockReplicatedLogEntry(1,
 251                 2, new MockRaftActorContext.MockPayload("2")));
 252
 253         sendMessageToSupport(new DeleteEntries(1));
 254
 255         assertEquals("Journal log size", 1, context.getReplicatedLog().size());
 256         assertEquals("Last index", 0, context.getReplicatedLog().lastIndex());
 257     }
 258
 259     @Test
 260     public void testUpdateElectionTerm() {
 261
 262         sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
 263
 264         assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
 265         assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
 266     }
 267
 268     @SuppressWarnings("unchecked")
 269     @Test
 270     public void testDataRecoveredWithPersistenceDisabled() {
 271         doNothing().when(mockCohort).applyRecoverySnapshot(aryEq(new byte[0]));
 272         doReturn(false).when(mockPersistence).isRecoveryApplicable();
 273         doReturn(10L).when(mockPersistentProvider).getLastSequenceNumber();
 274
 275         Snapshot snapshot = Snapshot.create(new byte[]{1}, Collections.<ReplicatedLogEntry>emptyList(), 3, 1, 3, 1);
 276         SnapshotOffer snapshotOffer = new SnapshotOffer(new SnapshotMetadata("test", 6, 12345), snapshot);
 277
 278         sendMessageToSupport(snapshotOffer);
 279
 280         sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
 281
 282         sendMessageToSupport(new MockRaftActorContext.MockReplicatedLogEntry(1,
 283                 4, new MockRaftActorContext.MockPayload("4")));
 284         sendMessageToSupport(new MockRaftActorContext.MockReplicatedLogEntry(1,
 285                 5, new MockRaftActorContext.MockPayload("5")));
 286
 287         sendMessageToSupport(new ApplyJournalEntries(4));
 288
 289         sendMessageToSupport(new DeleteEntries(5));
 290
 291         assertEquals("Journal log size", 0, context.getReplicatedLog().size());
 292         assertEquals("Last index", -1, context.getReplicatedLog().lastIndex());
 293         assertEquals("Last applied", -1, context.getLastApplied());
 294         assertEquals("Commit index", -1, context.getCommitIndex());
 295         assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
 296         assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
 297
 298         assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
 299         assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
 300
 301         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 302
 303         verify(mockCohort).applyRecoverySnapshot(aryEq(new byte[0]));
 304         verify(mockCohort, never()).getRestoreFromSnapshot();
 305         verifyNoMoreInteractions(mockCohort);
 306
 307         verify(mockPersistentProvider).deleteMessages(10L);
 308     }
 309
 310     static UpdateElectionTerm updateElectionTerm(final long term, final String votedFor) {
 311         return Matchers.argThat(new ArgumentMatcher<UpdateElectionTerm>() {
 312             @Override
 313             public boolean matches(Object argument) {
 314                 UpdateElectionTerm other = (UpdateElectionTerm) argument;
 315                 return term == other.getCurrentTerm() && votedFor.equals(other.getVotedFor());
 316             }
 317
 318             @Override
 319             public void describeTo(Description description) {
 320                 description.appendValue(new UpdateElectionTerm(term, votedFor));
 321             }
 322         });
 323     }
 324
 325     @Test
 326     public void testNoDataRecoveredWithPersistenceDisabled() {
 327         doReturn(false).when(mockPersistence).isRecoveryApplicable();
 328
 329         sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
 330
 331         assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
 332         assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
 333
 334         sendMessageToSupport(RecoveryCompleted.getInstance(), true);
 335
 336         verify(mockCohort).getRestoreFromSnapshot();
 337         verifyNoMoreInteractions(mockCohort, mockPersistentProvider);
 338     }
 339
 340     @Test
 341     public void testServerConfigurationPayloadApplied() {
 342         String follower1 = "follower1";
 343         String follower2 = "follower2";
 344         String follower3 = "follower3";
 345
 346         context.addToPeers(follower1, null, VotingState.VOTING);
 347         context.addToPeers(follower2, null, VotingState.VOTING);
 348
 349         //add new Server
 350         ServerConfigurationPayload obj = new ServerConfigurationPayload(Arrays.asList(
 351                 new ServerInfo(localId, true),
 352                 new ServerInfo(follower1, true),
 353                 new ServerInfo(follower2, false),
 354                 new ServerInfo(follower3, true)));
 355
 356         sendMessageToSupport(new MockRaftActorContext.MockReplicatedLogEntry(1, 0, obj));
 357
 358         //verify new peers
 359         assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 360         assertEquals("New peer Ids", Sets.newHashSet(follower1, follower2, follower3),
 361                 Sets.newHashSet(context.getPeerIds()));
 362         assertEquals("follower1 isVoting", true, context.getPeerInfo(follower1).isVoting());
 363         assertEquals("follower2 isVoting", false, context.getPeerInfo(follower2).isVoting());
 364         assertEquals("follower3 isVoting", true, context.getPeerInfo(follower3).isVoting());
 365
 366         sendMessageToSupport(new ApplyJournalEntries(0));
 367
 368         verify(mockCohort, never()).startLogRecoveryBatch(anyInt());
 369         verify(mockCohort, never()).appendRecoveredLogEntry(any(Payload.class));
 370
 371         //remove existing follower1
 372         obj = new ServerConfigurationPayload(Arrays.asList(
 373                 new ServerInfo(localId, true),
 374                 new ServerInfo("follower2", true),
 375                 new ServerInfo("follower3", true)));
 376
 377         sendMessageToSupport(new MockRaftActorContext.MockReplicatedLogEntry(1, 1, obj));
 378
 379         //verify new peers
 380         assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 381         assertEquals("New peer Ids", Sets.newHashSet(follower2, follower3), Sets.newHashSet(context.getPeerIds()));
 382     }
 383
 384     @Test
 385     public void testServerConfigurationPayloadAppliedWithPersistenceDisabled() {
 386         doReturn(false).when(mockPersistence).isRecoveryApplicable();
 387
 388         String follower = "follower";
 389         ServerConfigurationPayload obj = new ServerConfigurationPayload(Arrays.asList(
 390                 new ServerInfo(localId, true), new ServerInfo(follower, true)));
 391
 392         sendMessageToSupport(new MockRaftActorContext.MockReplicatedLogEntry(1, 0, obj));
 393
 394         //verify new peers
 395         assertEquals("New peer Ids", Sets.newHashSet(follower), Sets.newHashSet(context.getPeerIds()));
 396     }
 397
 398     @Test
 399     public void testOnSnapshotOfferWithServerConfiguration() {
 400         long electionTerm = 2;
 401         String electionVotedFor = "member-2";
 402         ServerConfigurationPayload serverPayload = new ServerConfigurationPayload(Arrays.asList(
 403                                                         new ServerInfo(localId, true),
 404                                                         new ServerInfo("follower1", true),
 405                                                         new ServerInfo("follower2", true)));
 406
 407         Snapshot snapshot = Snapshot.create(new byte[]{1}, Collections.<ReplicatedLogEntry>emptyList(),
 408                 -1, -1, -1, -1, electionTerm, electionVotedFor, serverPayload);
 409
 410         SnapshotMetadata metadata = new SnapshotMetadata("test", 6, 12345);
 411         SnapshotOffer snapshotOffer = new SnapshotOffer(metadata , snapshot);
 412
 413         sendMessageToSupport(snapshotOffer);
 414
 415         assertEquals("Journal log size", 0, context.getReplicatedLog().size());
 416         assertEquals("Election term", electionTerm, context.getTermInformation().getCurrentTerm());
 417         assertEquals("Election votedFor", electionVotedFor, context.getTermInformation().getVotedFor());
 418         assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
 419         assertEquals("Peer List", Sets.newHashSet("follower1", "follower2"),
 420             Sets.newHashSet(context.getPeerIds()));
 421     }
 422 }