Simplify code using Java 8 features
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / shardmanager / ShardManagerTest.java
1 /*
2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
10
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
21
22 import akka.actor.ActorRef;
23 import akka.actor.ActorSystem;
24 import akka.actor.AddressFromURIString;
25 import akka.actor.Props;
26 import akka.actor.Status;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.cluster.ClusterEvent;
31 import akka.cluster.Member;
32 import akka.dispatch.Dispatchers;
33 import akka.dispatch.OnComplete;
34 import akka.japi.Creator;
35 import akka.pattern.Patterns;
36 import akka.persistence.RecoveryCompleted;
37 import akka.serialization.Serialization;
38 import akka.testkit.TestActorRef;
39 import akka.testkit.javadsl.TestKit;
40 import akka.util.Timeout;
41 import com.google.common.base.Function;
42 import com.google.common.base.Stopwatch;
43 import com.google.common.collect.ImmutableMap;
44 import com.google.common.collect.Lists;
45 import com.google.common.collect.Sets;
46 import com.google.common.util.concurrent.Uninterruptibles;
47 import java.net.URI;
48 import java.util.AbstractMap;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.Collections;
52 import java.util.HashMap;
53 import java.util.List;
54 import java.util.Map;
55 import java.util.Map.Entry;
56 import java.util.Set;
57 import java.util.concurrent.CountDownLatch;
58 import java.util.concurrent.TimeUnit;
59 import java.util.concurrent.TimeoutException;
60 import java.util.stream.Collectors;
61 import org.junit.Test;
62 import org.mockito.Mockito;
63 import org.opendaylight.controller.cluster.access.concepts.MemberName;
64 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
65 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
66 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
67 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
68 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
69 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
70 import org.opendaylight.controller.cluster.datastore.Shard;
71 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
72 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
73 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
74 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
75 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
76 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
77 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
78 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
79 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
80 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
81 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
82 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
83 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
84 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
85 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
86 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
87 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
88 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
89 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
90 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
91 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
92 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
93 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
94 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
95 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
96 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
97 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
98 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
99 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
100 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
101 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
102 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
103 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
104 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
105 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
106 import org.opendaylight.controller.cluster.raft.RaftState;
107 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
108 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
109 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
110 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
111 import org.opendaylight.controller.cluster.raft.messages.AddServer;
112 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
113 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
114 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
115 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
116 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
117 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
118 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
119 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
120 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
121 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
122 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
123 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
124 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
125 import org.slf4j.Logger;
126 import org.slf4j.LoggerFactory;
127 import scala.concurrent.Await;
128 import scala.concurrent.Future;
129 import scala.concurrent.duration.FiniteDuration;
130
131 public class ShardManagerTest extends AbstractShardManagerTest {
132     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
133     private static final MemberName MEMBER_2 = MemberName.forName("member-2");
134     private static final MemberName MEMBER_3 = MemberName.forName("member-3");
135
136     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
137
138     private ActorSystem newActorSystem(final String config) {
139         return newActorSystem("cluster-test", config);
140     }
141
142     private ActorRef newMockShardActor(final ActorSystem system, final String shardName, final String memberName) {
143         String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
144         if (system == getSystem()) {
145             return actorFactory.createActor(MessageCollectorActor.props(), name);
146         }
147
148         return system.actorOf(MessageCollectorActor.props(), name);
149     }
150
151     private Props newShardMgrProps() {
152         return newShardMgrProps(new MockConfiguration());
153     }
154
155     private static DatastoreContextFactory newDatastoreContextFactory(final DatastoreContext datastoreContext) {
156         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
157         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
158         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
159         return mockFactory;
160     }
161
162     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
163         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
164     }
165
166     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(final ActorRef shardActor) {
167         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
168                 .distributedDataStore(mock(DistributedDataStore.class));
169     }
170
171
172     private Props newPropsShardMgrWithMockShardActor() {
173         return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
174                 Dispatchers.DefaultDispatcherId());
175     }
176
177     private Props newPropsShardMgrWithMockShardActor(final ActorRef shardActor) {
178         return newTestShardMgrBuilderWithMockShardActor(shardActor).props()
179                 .withDispatcher(Dispatchers.DefaultDispatcherId());
180     }
181
182
183     private TestShardManager newTestShardManager() {
184         return newTestShardManager(newShardMgrProps());
185     }
186
187     private TestShardManager newTestShardManager(final Props props) {
188         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
189         TestShardManager shardManager = shardManagerActor.underlyingActor();
190         shardManager.waitForRecoveryComplete();
191         return shardManager;
192     }
193
194     private static void waitForShardInitialized(final ActorRef shardManager, final String shardName,
195             final TestKit kit) {
196         AssertionError last = null;
197         Stopwatch sw = Stopwatch.createStarted();
198         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
199             try {
200                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
201                 kit.expectMsgClass(LocalShardFound.class);
202                 return;
203             } catch (AssertionError e) {
204                 last = e;
205             }
206
207             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
208         }
209
210         throw last;
211     }
212
213     @SuppressWarnings("unchecked")
214     private static <T> T expectMsgClassOrFailure(final Class<T> msgClass, final TestKit kit, final String msg) {
215         Object reply = kit.expectMsgAnyClassOf(kit.duration("5 sec"), msgClass, Failure.class);
216         if (reply instanceof Failure) {
217             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
218         }
219
220         return (T)reply;
221     }
222
223     @Test
224     public void testPerShardDatastoreContext() throws Exception {
225         LOG.info("testPerShardDatastoreContext starting");
226         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
227                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
228
229         Mockito.doReturn(
230                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
231                 .when(mockFactory).getShardDatastoreContext("default");
232
233         Mockito.doReturn(
234                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
235                 .when(mockFactory).getShardDatastoreContext("topology");
236
237         final MockConfiguration mockConfig = new MockConfiguration() {
238             @Override
239             public Collection<String> getMemberShardNames(final MemberName memberName) {
240                 return Arrays.asList("default", "topology");
241             }
242
243             @Override
244             public Collection<MemberName> getMembersFromShardName(final String shardName) {
245                 return members("member-1");
246             }
247         };
248
249         final ActorRef defaultShardActor = actorFactory.createActor(
250                 MessageCollectorActor.props(), actorFactory.generateActorId("default"));
251         final ActorRef topologyShardActor = actorFactory.createActor(
252                 MessageCollectorActor.props(), actorFactory.generateActorId("topology"));
253
254         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
255                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
256         shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
257         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
258
259         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
260         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
261         class LocalShardManager extends ShardManager {
262             LocalShardManager(final AbstractShardManagerCreator<?> creator) {
263                 super(creator);
264             }
265
266             @Override
267             protected ActorRef newShardActor(final ShardInformation info) {
268                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
269                 ActorRef ref = null;
270                 if (entry != null) {
271                     ref = entry.getKey();
272                     entry.setValue(info.getDatastoreContext());
273                 }
274
275                 newShardActorLatch.countDown();
276                 return ref;
277             }
278         }
279
280         final Creator<ShardManager> creator = new Creator<ShardManager>() {
281             private static final long serialVersionUID = 1L;
282             @Override
283             public ShardManager create() {
284                 return new LocalShardManager(
285                         new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
286                                 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
287             }
288         };
289
290         TestKit kit = new TestKit(getSystem());
291
292         final ActorRef shardManager = actorFactory.createActor(Props.create(
293                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
294
295         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
296
297         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
298         assertEquals("getShardElectionTimeoutFactor", 6,
299                 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
300         assertEquals("getShardElectionTimeoutFactor", 7,
301                 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
302
303         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
304                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
305         Mockito.doReturn(
306                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
307                 .when(newMockFactory).getShardDatastoreContext("default");
308
309         Mockito.doReturn(
310                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
311                 .when(newMockFactory).getShardDatastoreContext("topology");
312
313         shardManager.tell(newMockFactory, kit.getRef());
314
315         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
316                 DatastoreContext.class);
317         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
318
319         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
320         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
321
322         LOG.info("testPerShardDatastoreContext ending");
323     }
324
325     @Test
326     public void testOnReceiveFindPrimaryForNonExistentShard() {
327         new TestKit(getSystem()) {
328             {
329                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
330
331                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
332
333                 shardManager.tell(new FindPrimary("non-existent", false), getRef());
334
335                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
336             }
337         };
338     }
339
340     @Test
341     public void testOnReceiveFindPrimaryForLocalLeaderShard() {
342         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
343         new TestKit(getSystem()) {
344             {
345                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
346
347                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
348
349                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
350                 shardManager.tell(new ActorInitialized(), mockShardActor);
351
352                 DataTree mockDataTree = mock(DataTree.class);
353                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
354                         DataStoreVersions.CURRENT_VERSION), getRef());
355
356                 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
357                 shardManager.tell(
358                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
359                         mockShardActor);
360
361                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
362
363                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
364                         LocalPrimaryShardFound.class);
365                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
366                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
367                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
368             }
369         };
370
371         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
372     }
373
374     @Test
375     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() {
376         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
377         new TestKit(getSystem()) {
378             {
379                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
380
381                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
382                 shardManager.tell(new ActorInitialized(), mockShardActor);
383
384                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
385                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
386                 shardManager.tell(
387                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
388                         mockShardActor);
389                 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
390                         mockShardActor);
391
392                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
393
394                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
395             }
396         };
397
398         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
399     }
400
401     @Test
402     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() {
403         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
404         new TestKit(getSystem()) {
405             {
406                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
407
408                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
409                 shardManager.tell(new ActorInitialized(), mockShardActor);
410
411                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
412                 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
413
414                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
415                 shardManager.tell(
416                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
417                         mockShardActor);
418                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
419                 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
420
421                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
422
423                 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
424                         RemotePrimaryShardFound.class);
425                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
426                         primaryFound.getPrimaryPath().contains("member-2-shard-default"));
427                 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
428             }
429         };
430
431         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
432     }
433
434     @Test
435     public void testOnReceiveFindPrimaryForUninitializedShard() {
436         new TestKit(getSystem()) {
437             {
438                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
439
440                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
441
442                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
443             }
444         };
445     }
446
447     @Test
448     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() {
449         new TestKit(getSystem()) {
450             {
451                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
452
453                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
454                 shardManager.tell(new ActorInitialized(), mockShardActor);
455
456                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
457
458                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
459             }
460         };
461     }
462
463     @Test
464     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() {
465         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
466         new TestKit(getSystem()) {
467             {
468                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
469
470                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
471                 shardManager.tell(new ActorInitialized(), mockShardActor);
472
473                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
474                 shardManager.tell(
475                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
476                         mockShardActor);
477
478                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
479
480                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
481
482                 DataTree mockDataTree = mock(DataTree.class);
483                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
484                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
485
486                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
487
488                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
489                         LocalPrimaryShardFound.class);
490                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
491                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
492                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
493             }
494         };
495
496         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
497     }
498
499     @Test
500     public void testOnReceiveFindPrimaryWaitForShardLeader() {
501         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
502         datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
503         new TestKit(getSystem()) {
504             {
505                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
506
507                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
508
509                 // We're passing waitUntilInitialized = true to FindPrimary so
510                 // the response should be
511                 // delayed until we send ActorInitialized and
512                 // RoleChangeNotification.
513                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
514
515                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
516
517                 shardManager.tell(new ActorInitialized(), mockShardActor);
518
519                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
520
521                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
522                 shardManager.tell(
523                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
524                         mockShardActor);
525
526                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
527
528                 DataTree mockDataTree = mock(DataTree.class);
529                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
530                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
531
532                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
533                         LocalPrimaryShardFound.class);
534                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
535                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
536                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
537
538                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
539             }
540         };
541
542         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
543     }
544
545     @Test
546     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() {
547         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
548         new TestKit(getSystem()) {
549             {
550                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
551
552                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
553
554                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
555
556                 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
557
558                 shardManager.tell(new ActorInitialized(), mockShardActor);
559
560                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
561             }
562         };
563
564         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
565     }
566
567     @Test
568     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() {
569         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
570         new TestKit(getSystem()) {
571             {
572                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
573
574                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
575                 shardManager.tell(new ActorInitialized(), mockShardActor);
576                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
577                         RaftState.Candidate.name()), mockShardActor);
578
579                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
580
581                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
582             }
583         };
584
585         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
586     }
587
588     @Test
589     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() {
590         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
591         new TestKit(getSystem()) {
592             {
593                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
594
595                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
596                 shardManager.tell(new ActorInitialized(), mockShardActor);
597                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
598                         RaftState.IsolatedLeader.name()), mockShardActor);
599
600                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
601
602                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
603             }
604         };
605
606         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
607     }
608
609     @Test
610     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() {
611         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
612         new TestKit(getSystem()) {
613             {
614                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
615
616                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
617                 shardManager.tell(new ActorInitialized(), mockShardActor);
618
619                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
620
621                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
622             }
623         };
624
625         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
626     }
627
628     @Test
629     public void testOnReceiveFindPrimaryForRemoteShard() {
630         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
631         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
632
633         // Create an ActorSystem ShardManager actor for member-1.
634
635         final ActorSystem system1 = newActorSystem("Member1");
636         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
637
638         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
639                 newTestShardMgrBuilderWithMockShardActor().cluster(
640                         new ClusterWrapperImpl(system1)).props().withDispatcher(
641                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
642
643         // Create an ActorSystem ShardManager actor for member-2.
644
645         final ActorSystem system2 = newActorSystem("Member2");
646
647         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
648
649         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
650
651         MockConfiguration mockConfig2 = new MockConfiguration(
652                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
653                         .put("astronauts", Arrays.asList("member-2")).build());
654
655         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
656                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
657                         new ClusterWrapperImpl(system2)).props().withDispatcher(
658                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
659
660         new TestKit(system1) {
661             {
662                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
663                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
664
665                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
666
667                 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
668                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
669                 shardManager2.tell(
670                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
671                         mockShardActor2);
672                 shardManager2.tell(
673                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
674                         mockShardActor2);
675
676                 shardManager1.underlyingActor().waitForMemberUp();
677                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
678
679                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
680                 String path = found.getPrimaryPath();
681                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
682                 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
683
684                 shardManager2.underlyingActor().verifyFindPrimary();
685
686                 // This part times out quite a bit on jenkins for some reason
687
688 //                Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
689 //
690 //                shardManager1.underlyingActor().waitForMemberRemoved();
691 //
692 //                shardManager1.tell(new FindPrimary("astronauts", false), getRef());
693 //
694 //                expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
695             }
696         };
697
698         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
699     }
700
701     @Test
702     public void testShardAvailabilityOnChangeOfMemberReachability() {
703         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
704         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
705
706         // Create an ActorSystem ShardManager actor for member-1.
707
708         final ActorSystem system1 = newActorSystem("Member1");
709         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
710
711         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
712
713         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
714                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
715                         new ClusterWrapperImpl(system1)).props().withDispatcher(
716                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
717
718         // Create an ActorSystem ShardManager actor for member-2.
719
720         final ActorSystem system2 = newActorSystem("Member2");
721
722         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
723
724         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
725
726         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
727                 .put("default", Arrays.asList("member-1", "member-2")).build());
728
729         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
730                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
731                         new ClusterWrapperImpl(system2)).props().withDispatcher(
732                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
733
734         new TestKit(system1) {
735             {
736                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
737                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
738                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
739                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
740
741                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
742                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
743                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
744                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
745                 shardManager1.tell(
746                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
747                         mockShardActor1);
748                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
749                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
750                 shardManager2.tell(
751                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
752                         mockShardActor2);
753                 shardManager1.underlyingActor().waitForMemberUp();
754
755                 shardManager1.tell(new FindPrimary("default", true), getRef());
756
757                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
758                 String path = found.getPrimaryPath();
759                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
760
761                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
762                         "akka://cluster-test@127.0.0.1:2558"), getRef());
763
764                 shardManager1.underlyingActor().waitForUnreachableMember();
765
766                 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
767                 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
768                 MessageCollectorActor.clearMessages(mockShardActor1);
769
770                 shardManager1.tell(
771                         MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
772                         getRef());
773
774                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
775
776                 shardManager1.tell(new FindPrimary("default", true), getRef());
777
778                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
779
780                 shardManager1.tell(
781                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
782                         getRef());
783
784                 shardManager1.underlyingActor().waitForReachableMember();
785
786                 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
787                 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
788                 MessageCollectorActor.clearMessages(mockShardActor1);
789
790                 shardManager1.tell(new FindPrimary("default", true), getRef());
791
792                 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
793                 String path1 = found1.getPrimaryPath();
794                 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
795
796                 shardManager1.tell(
797                         MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
798                         getRef());
799
800                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
801
802                 // Test FindPrimary wait succeeds after reachable member event.
803
804                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
805                         "akka://cluster-test@127.0.0.1:2558"), getRef());
806                 shardManager1.underlyingActor().waitForUnreachableMember();
807
808                 shardManager1.tell(new FindPrimary("default", true), getRef());
809
810                 shardManager1.tell(
811                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
812                         getRef());
813
814                 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
815                 String path2 = found2.getPrimaryPath();
816                 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
817             }
818         };
819
820         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
821     }
822
823     @Test
824     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() {
825         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
826         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
827
828         // Create an ActorSystem ShardManager actor for member-1.
829
830         final ActorSystem system1 = newActorSystem("Member1");
831         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
832
833         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
834
835         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
836         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
837                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
838                         .primaryShardInfoCache(primaryShardInfoCache).props()
839                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
840                 shardManagerID);
841
842         // Create an ActorSystem ShardManager actor for member-2.
843
844         final ActorSystem system2 = newActorSystem("Member2");
845
846         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
847
848         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
849
850         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
851                 .put("default", Arrays.asList("member-1", "member-2")).build());
852
853         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
854                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
855                         new ClusterWrapperImpl(system2)).props().withDispatcher(
856                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
857
858         new TestKit(system1) {
859             {
860                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
861                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
862                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
863                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
864
865                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
866                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
867                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
868                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
869                 shardManager1.tell(
870                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
871                         mockShardActor1);
872                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
873                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
874                 shardManager2.tell(
875                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
876                         mockShardActor2);
877                 shardManager1.underlyingActor().waitForMemberUp();
878
879                 shardManager1.tell(new FindPrimary("default", true), getRef());
880
881                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
882                 String path = found.getPrimaryPath();
883                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
884
885                 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
886                         system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
887
888                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
889                         "akka://cluster-test@127.0.0.1:2558"), getRef());
890
891                 shardManager1.underlyingActor().waitForUnreachableMember();
892
893                 shardManager1.tell(new FindPrimary("default", true), getRef());
894
895                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
896
897                 assertNull("Expected primaryShardInfoCache entry removed",
898                         primaryShardInfoCache.getIfPresent("default"));
899
900                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
901                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
902                 shardManager1.tell(
903                         new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
904                         mockShardActor1);
905
906                 shardManager1.tell(new FindPrimary("default", true), getRef());
907
908                 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
909                 String path1 = found1.getPrimaryPath();
910                 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
911
912             }
913         };
914
915         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
916     }
917
918     @Test
919     public void testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable() {
920         LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable starting");
921         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
922
923         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
924                 .put("default", Arrays.asList("member-256", "member-2")).build());
925
926         // Create an ActorSystem, ShardManager and actor for member-256.
927
928         final ActorSystem system256 = newActorSystem("Member256");
929         // 2562 is the tcp port of Member256 in src/test/resources/application.conf.
930         Cluster.get(system256).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
931
932         final ActorRef mockShardActor256 = newMockShardActor(system256, Shard.DEFAULT_NAME, "member-256");
933
934         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
935
936         // ShardManager must be created with shard configuration to let its localShards has shards.
937         final TestActorRef<TestShardManager> shardManager256 = TestActorRef.create(system256,
938                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor256)
939                         .cluster(new ClusterWrapperImpl(system256))
940                         .primaryShardInfoCache(primaryShardInfoCache).props()
941                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
942                 shardManagerID);
943
944         // Create an ActorSystem, ShardManager and actor for member-2 whose name is contained in member-256.
945
946         final ActorSystem system2 = newActorSystem("Member2");
947
948         // Join member-2 into the cluster of member-256.
949         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
950
951         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
952
953         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
954                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor2).cluster(
955                         new ClusterWrapperImpl(system2)).props().withDispatcher(
956                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
957
958         new TestKit(system256) {
959             {
960                 shardManager256.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
961                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
962                 shardManager256.tell(new ActorInitialized(), mockShardActor256);
963                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
964
965                 String memberId256 = "member-256-shard-default-" + shardMrgIDSuffix;
966                 String memberId2   = "member-2-shard-default-"   + shardMrgIDSuffix;
967                 shardManager256.tell(new ShardLeaderStateChanged(memberId256, memberId256, mock(DataTree.class),
968                         DataStoreVersions.CURRENT_VERSION), mockShardActor256);
969                 shardManager256.tell(
970                         new RoleChangeNotification(memberId256, RaftState.Candidate.name(), RaftState.Leader.name()),
971                         mockShardActor256);
972                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId256, mock(DataTree.class),
973                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
974                 shardManager2.tell(
975                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Follower.name()),
976                         mockShardActor2);
977                 shardManager256.underlyingActor().waitForMemberUp();
978
979                 shardManager256.tell(new FindPrimary("default", true), getRef());
980
981                 LocalPrimaryShardFound found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
982                 String path = found.getPrimaryPath();
983                 assertTrue("Unexpected primary path " + path + " which must on member-256",
984                             path.contains("member-256-shard-default-config"));
985
986                 PrimaryShardInfo primaryShardInfo = new PrimaryShardInfo(
987                         system256.actorSelection(mockShardActor256.path()), DataStoreVersions.CURRENT_VERSION);
988                 primaryShardInfoCache.putSuccessful("default", primaryShardInfo);
989
990                 // Simulate member-2 become unreachable.
991                 shardManager256.tell(MockClusterWrapper.createUnreachableMember("member-2",
992                         "akka://cluster-test@127.0.0.1:2558"), getRef());
993                 shardManager256.underlyingActor().waitForUnreachableMember();
994
995                 // Make sure leader shard on member-256 is still leader and still in the cache.
996                 shardManager256.tell(new FindPrimary("default", true), getRef());
997                 found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
998                 path = found.getPrimaryPath();
999                 assertTrue("Unexpected primary path " + path + " which must still not on member-256",
1000                             path.contains("member-256-shard-default-config"));
1001                 Future<PrimaryShardInfo> futurePrimaryShard = primaryShardInfoCache.getIfPresent("default");
1002                 futurePrimaryShard.onComplete(new OnComplete<PrimaryShardInfo>() {
1003                     @Override
1004                     public void onComplete(final Throwable failure, final PrimaryShardInfo futurePrimaryShardInfo) {
1005                         if (failure != null) {
1006                             assertTrue("Primary shard info is unexpectedly removed from primaryShardInfoCache", false);
1007                         } else {
1008                             assertEquals("Expected primaryShardInfoCache entry",
1009                                         primaryShardInfo, futurePrimaryShardInfo);
1010                         }
1011                     }
1012                 }, system256.dispatchers().defaultGlobalDispatcher());
1013             }
1014         };
1015
1016         LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable ending");
1017     }
1018
1019     @Test
1020     public void testOnReceiveFindLocalShardForNonExistentShard() {
1021         new TestKit(getSystem()) {
1022             {
1023                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1024
1025                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1026
1027                 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
1028
1029                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1030
1031                 assertEquals("getShardName", "non-existent", notFound.getShardName());
1032             }
1033         };
1034     }
1035
1036     @Test
1037     public void testOnReceiveFindLocalShardForExistentShard() {
1038         new TestKit(getSystem()) {
1039             {
1040                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1041
1042                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1043                 shardManager.tell(new ActorInitialized(), mockShardActor);
1044
1045                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1046
1047                 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1048
1049                 assertTrue("Found path contains " + found.getPath().path().toString(),
1050                         found.getPath().path().toString().contains("member-1-shard-default-config"));
1051             }
1052         };
1053     }
1054
1055     @Test
1056     public void testOnReceiveFindLocalShardForNotInitializedShard() {
1057         new TestKit(getSystem()) {
1058             {
1059                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1060
1061                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1062
1063                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1064             }
1065         };
1066     }
1067
1068     @Test
1069     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
1070         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1071         new TestKit(getSystem()) {
1072             {
1073                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1074
1075                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1076
1077                 // We're passing waitUntilInitialized = true to FindLocalShard
1078                 // so the response should be
1079                 // delayed until we send ActorInitialized.
1080                 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
1081                         new Timeout(5, TimeUnit.SECONDS));
1082
1083                 shardManager.tell(new ActorInitialized(), mockShardActor);
1084
1085                 Object resp = Await.result(future, duration("5 seconds"));
1086                 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
1087             }
1088         };
1089
1090         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1091     }
1092
1093     @Test
1094     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1095         TestShardManager shardManager = newTestShardManager();
1096
1097         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1098         shardManager.onReceiveCommand(new RoleChangeNotification(
1099                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1100
1101         verify(ready, never()).countDown();
1102
1103         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1104                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1105
1106         verify(ready, times(1)).countDown();
1107     }
1108
1109     @Test
1110     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1111         new TestKit(getSystem()) {
1112             {
1113                 TestShardManager shardManager = newTestShardManager();
1114
1115                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1116                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1117
1118                 verify(ready, never()).countDown();
1119
1120                 shardManager
1121                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1122
1123                 shardManager.onReceiveCommand(
1124                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1125                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1126
1127                 verify(ready, times(1)).countDown();
1128             }
1129         };
1130     }
1131
1132     @Test
1133     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1134         new TestKit(getSystem()) {
1135             {
1136                 TestShardManager shardManager = newTestShardManager();
1137
1138                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1139                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1140
1141                 verify(ready, never()).countDown();
1142
1143                 shardManager.onReceiveCommand(
1144                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1145                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1146
1147                 shardManager
1148                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1149
1150                 verify(ready, times(1)).countDown();
1151             }
1152         };
1153     }
1154
1155     @Test
1156     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1157         TestShardManager shardManager = newTestShardManager();
1158
1159         shardManager.onReceiveCommand(new RoleChangeNotification(
1160                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1161
1162         verify(ready, never()).countDown();
1163     }
1164
1165     @Test
1166     public void testByDefaultSyncStatusIsFalse() {
1167         TestShardManager shardManager = newTestShardManager();
1168
1169         assertEquals(false, shardManager.getMBean().getSyncStatus());
1170     }
1171
1172     @Test
1173     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1174         TestShardManager shardManager = newTestShardManager();
1175
1176         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1177                 RaftState.Follower.name(), RaftState.Leader.name()));
1178
1179         assertEquals(true, shardManager.getMBean().getSyncStatus());
1180     }
1181
1182     @Test
1183     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1184         TestShardManager shardManager = newTestShardManager();
1185
1186         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1187         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1188                 RaftState.Follower.name(), RaftState.Candidate.name()));
1189
1190         assertEquals(false, shardManager.getMBean().getSyncStatus());
1191
1192         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1193         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1194                 true, shardId));
1195
1196         assertEquals(false, shardManager.getMBean().getSyncStatus());
1197     }
1198
1199     @Test
1200     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1201         TestShardManager shardManager = newTestShardManager();
1202
1203         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1204         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1205                 RaftState.Candidate.name(), RaftState.Follower.name()));
1206
1207         // Initially will be false
1208         assertEquals(false, shardManager.getMBean().getSyncStatus());
1209
1210         // Send status true will make sync status true
1211         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1212
1213         assertEquals(true, shardManager.getMBean().getSyncStatus());
1214
1215         // Send status false will make sync status false
1216         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1217
1218         assertEquals(false, shardManager.getMBean().getSyncStatus());
1219
1220     }
1221
1222     @Test
1223     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1224         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1225         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1226             @Override
1227             public List<String> getMemberShardNames(final MemberName memberName) {
1228                 return Arrays.asList("default", "astronauts");
1229             }
1230         }));
1231
1232         // Initially will be false
1233         assertEquals(false, shardManager.getMBean().getSyncStatus());
1234
1235         // Make default shard leader
1236         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1237         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1238                 RaftState.Follower.name(), RaftState.Leader.name()));
1239
1240         // default = Leader, astronauts is unknown so sync status remains false
1241         assertEquals(false, shardManager.getMBean().getSyncStatus());
1242
1243         // Make astronauts shard leader as well
1244         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1245         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1246                 RaftState.Follower.name(), RaftState.Leader.name()));
1247
1248         // Now sync status should be true
1249         assertEquals(true, shardManager.getMBean().getSyncStatus());
1250
1251         // Make astronauts a Follower
1252         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1253                 RaftState.Leader.name(), RaftState.Follower.name()));
1254
1255         // Sync status is not true
1256         assertEquals(false, shardManager.getMBean().getSyncStatus());
1257
1258         // Make the astronauts follower sync status true
1259         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1260
1261         // Sync status is now true
1262         assertEquals(true, shardManager.getMBean().getSyncStatus());
1263
1264         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1265     }
1266
1267     @Test
1268     public void testOnReceiveSwitchShardBehavior() {
1269         new TestKit(getSystem()) {
1270             {
1271                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1272
1273                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1274                 shardManager.tell(new ActorInitialized(), mockShardActor);
1275
1276                 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1277
1278                 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1279                         SwitchBehavior.class);
1280
1281                 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1282                 assertEquals(1000, switchBehavior.getNewTerm());
1283             }
1284         };
1285     }
1286
1287     private static List<MemberName> members(final String... names) {
1288         return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1289     }
1290
1291     @Test
1292     public void testOnCreateShard() {
1293         LOG.info("testOnCreateShard starting");
1294         new TestKit(getSystem()) {
1295             {
1296                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1297
1298                 ActorRef shardManager = actorFactory
1299                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1300                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1301
1302                 SchemaContext schemaContext = TestModel.createTestContext();
1303                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1304
1305                 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1306                         .persistent(false).build();
1307                 Shard.Builder shardBuilder = Shard.builder();
1308
1309                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1310                         "foo", null, members("member-1", "member-5", "member-6"));
1311                 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1312
1313                 expectMsgClass(duration("5 seconds"), Success.class);
1314
1315                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1316
1317                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1318
1319                 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1320                 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1321                         .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1322                 assertEquals("peerMembers", Sets.newHashSet(
1323                         ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1324                         ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1325                         shardBuilder.getPeerAddresses().keySet());
1326                 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1327                         shardBuilder.getId());
1328                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1329
1330                 // Send CreateShard with same name - should return Success with
1331                 // a message.
1332
1333                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1334
1335                 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1336                 assertNotNull("Success status is null", success.status());
1337             }
1338         };
1339
1340         LOG.info("testOnCreateShard ending");
1341     }
1342
1343     @Test
1344     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1345         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1346         new TestKit(getSystem()) {
1347             {
1348                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1349
1350                 ActorRef shardManager = actorFactory
1351                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1352                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1353
1354                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1355
1356                 Shard.Builder shardBuilder = Shard.builder();
1357                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1358                         "foo", null, members("member-5", "member-6"));
1359
1360                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1361                 expectMsgClass(duration("5 seconds"), Success.class);
1362
1363                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1364                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1365
1366                 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1367                 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1368                         .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1369             }
1370         };
1371
1372         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1373     }
1374
1375     @Test
1376     public void testOnCreateShardWithNoInitialSchemaContext() {
1377         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1378         new TestKit(getSystem()) {
1379             {
1380                 ActorRef shardManager = actorFactory
1381                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1382                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1383
1384                 Shard.Builder shardBuilder = Shard.builder();
1385
1386                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1387                         "foo", null, members("member-1"));
1388                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1389
1390                 expectMsgClass(duration("5 seconds"), Success.class);
1391
1392                 SchemaContext schemaContext = TestModel.createTestContext();
1393                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1394
1395                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1396
1397                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1398
1399                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1400                 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1401             }
1402         };
1403
1404         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1405     }
1406
1407     @Test
1408     public void testGetSnapshot() {
1409         LOG.info("testGetSnapshot starting");
1410         TestKit kit = new TestKit(getSystem());
1411
1412         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1413                 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1414                 .put("astronauts", Collections.<String>emptyList()).build());
1415
1416         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig)
1417                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1418
1419         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1420         Failure failure = kit.expectMsgClass(Failure.class);
1421         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1422
1423         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1424
1425         waitForShardInitialized(shardManager, "shard1", kit);
1426         waitForShardInitialized(shardManager, "shard2", kit);
1427
1428         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1429
1430         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1431
1432         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1433         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1434
1435         Function<ShardSnapshot, String> shardNameTransformer = ShardSnapshot::getName;
1436
1437         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1438             datastoreSnapshot.getShardSnapshots().stream().map(shardNameTransformer).collect(Collectors.toSet())));
1439
1440         // Add a new replica
1441
1442         TestKit mockShardLeaderKit = new TestKit(getSystem());
1443
1444         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1445         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1446
1447         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1448         mockShardLeaderKit.expectMsgClass(AddServer.class);
1449         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1450         kit.expectMsgClass(Status.Success.class);
1451         waitForShardInitialized(shardManager, "astronauts", kit);
1452
1453         // Send another GetSnapshot and verify
1454
1455         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1456         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1457
1458         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1459                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1460
1461         ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1462         assertNotNull("Expected ShardManagerSnapshot", snapshot);
1463         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1464                 Sets.newHashSet(snapshot.getShardList()));
1465
1466         LOG.info("testGetSnapshot ending");
1467     }
1468
1469     @Test
1470     public void testRestoreFromSnapshot() {
1471         LOG.info("testRestoreFromSnapshot starting");
1472
1473         datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1474
1475         TestKit kit = new TestKit(getSystem());
1476
1477         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1478                 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1479                 .put("astronauts", Collections.<String>emptyList()).build());
1480
1481         ShardManagerSnapshot snapshot =
1482                 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1483         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1484                 Collections.<ShardSnapshot>emptyList());
1485         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1486                 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1487
1488         shardManager.underlyingActor().waitForRecoveryComplete();
1489
1490         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1491
1492         waitForShardInitialized(shardManager, "shard1", kit);
1493         waitForShardInitialized(shardManager, "shard2", kit);
1494         waitForShardInitialized(shardManager, "astronauts", kit);
1495
1496         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1497
1498         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1499
1500         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1501
1502         assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1503         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1504                 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1505
1506         LOG.info("testRestoreFromSnapshot ending");
1507     }
1508
1509     @Test
1510     public void testAddShardReplicaForNonExistentShardConfig() {
1511         new TestKit(getSystem()) {
1512             {
1513                 ActorRef shardManager = actorFactory
1514                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1515                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1516
1517                 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1518                 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1519
1520                 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1521             }
1522         };
1523     }
1524
1525     @Test
1526     public void testAddShardReplica() {
1527         LOG.info("testAddShardReplica starting");
1528         MockConfiguration mockConfig = new MockConfiguration(
1529                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1530                         .put("astronauts", Arrays.asList("member-2")).build());
1531
1532         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1533         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1534
1535         // Create an ActorSystem ShardManager actor for member-1.
1536         final ActorSystem system1 = newActorSystem("Member1");
1537         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1538         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1539         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1540                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1541                         .cluster(new ClusterWrapperImpl(system1)).props()
1542                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1543                 shardManagerID);
1544
1545         // Create an ActorSystem ShardManager actor for member-2.
1546         final ActorSystem system2 = newActorSystem("Member2");
1547         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1548
1549         String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1550         String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1551         final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1552                 Props.create(MockRespondActor.class, AddServer.class,
1553                         new AddServerReply(ServerChangeStatus.OK, memberId2))
1554                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1555                 name);
1556         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1557                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1558                         .cluster(new ClusterWrapperImpl(system2)).props()
1559                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1560                 shardManagerID);
1561
1562         new TestKit(system1) {
1563             {
1564                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1565                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1566
1567                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1568
1569                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1570                 leaderShardManager.tell(
1571                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1572                         mockShardLeaderActor);
1573                 leaderShardManager.tell(
1574                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1575                         mockShardLeaderActor);
1576
1577                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1578                 leaderShardManager.underlyingActor().waitForMemberUp();
1579
1580                 // Have a dummy snapshot to be overwritten by the new data
1581                 // persisted.
1582                 String[] restoredShards = { "default", "people" };
1583                 ShardManagerSnapshot snapshot =
1584                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1585                 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1586                 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1587
1588                 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1589                 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1590
1591                 // construct a mock response message
1592                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1593                 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1594                         AddServer.class);
1595                 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1596                 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1597                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1598
1599                 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1600                 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1601                 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1602                         ShardManagerSnapshot.class);
1603                 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1604                 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1605                 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1606                         Sets.newHashSet(shardManagerSnapshot.getShardList()));
1607             }
1608         };
1609         LOG.info("testAddShardReplica ending");
1610     }
1611
1612     @Test
1613     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() {
1614         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1615         new TestKit(getSystem()) {
1616             {
1617                 TestActorRef<TestShardManager> shardManager = actorFactory
1618                         .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1619
1620                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1621                 shardManager.tell(new ActorInitialized(), mockShardActor);
1622
1623                 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1624                 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1625                 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1626                         .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1627
1628                 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1629
1630                 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1631                 shardManager.tell(
1632                         new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1633                         mockShardActor);
1634                 shardManager.tell(
1635                         new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1636                         mockShardActor);
1637
1638                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1639
1640                 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1641
1642                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1643                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1644
1645                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1646                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1647
1648                 // Send message again to verify previous in progress state is
1649                 // cleared
1650
1651                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1652                 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1653                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1654
1655                 // Send message again with an AddServer timeout to verify the
1656                 // pre-existing shard actor isn't terminated.
1657
1658                 shardManager.tell(
1659                         newDatastoreContextFactory(
1660                                 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1661                         getRef());
1662                 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1663                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1664                 expectMsgClass(duration("5 seconds"), Failure.class);
1665
1666                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1667                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1668             }
1669         };
1670
1671         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1672     }
1673
1674     @Test
1675     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() {
1676         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1677         new TestKit(getSystem()) {
1678             {
1679                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1680                 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1681
1682                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1683                 shardManager.tell(new ActorInitialized(), mockShardActor);
1684                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1685                         DataStoreVersions.CURRENT_VERSION), getRef());
1686                 shardManager.tell(
1687                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1688                         mockShardActor);
1689
1690                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1691                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1692                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1693
1694                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1695                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1696             }
1697         };
1698
1699         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1700     }
1701
1702     @Test
1703     public void testAddShardReplicaWithAddServerReplyFailure() {
1704         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1705         new TestKit(getSystem()) {
1706             {
1707                 TestKit mockShardLeaderKit = new TestKit(getSystem());
1708
1709                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1710                         .put("astronauts", Arrays.asList("member-2")).build());
1711
1712                 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1713                 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1714                         newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props()
1715                             .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1716                 shardManager.underlyingActor()
1717                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1718
1719                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1720
1721                 TestKit terminateWatcher = new TestKit(getSystem());
1722                 terminateWatcher.watch(mockNewReplicaShardActor);
1723
1724                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1725
1726                 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1727                 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1728                         addServerMsg.getNewServerId());
1729                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1730
1731                 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1732                 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1733
1734                 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1735                 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1736
1737                 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1738
1739                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1740                 mockShardLeaderKit.expectMsgClass(AddServer.class);
1741                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1742                 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1743                 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1744             }
1745         };
1746
1747         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1748     }
1749
1750     @Test
1751     public void testAddShardReplicaWithAlreadyInProgress() {
1752         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1753                 AddServer.class, new AddShardReplica("astronauts"));
1754     }
1755
1756     @Test
1757     public void testAddShardReplicaWithFindPrimaryTimeout() {
1758         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1759         datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1760         new TestKit(getSystem()) {
1761             {
1762                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1763                         .put("astronauts", Arrays.asList("member-2")).build());
1764
1765                 final ActorRef newReplicaShardManager = actorFactory
1766                         .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props()
1767                                 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1768
1769                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1770                 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1771                         AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1772
1773                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1774                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1775                 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1776             }
1777         };
1778
1779         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1780     }
1781
1782     @Test
1783     public void testRemoveShardReplicaForNonExistentShard() {
1784         new TestKit(getSystem()) {
1785             {
1786                 ActorRef shardManager = actorFactory
1787                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1788                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1789
1790                 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1791                 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1792                 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1793             }
1794         };
1795     }
1796
1797     @Test
1798     /**
1799      * Primary is Local.
1800      */
1801     public void testRemoveShardReplicaLocal() {
1802         new TestKit(getSystem()) {
1803             {
1804                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1805
1806                 final ActorRef respondActor = actorFactory.createActor(Props.create(MockRespondActor.class,
1807                         RemoveServer.class, new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1808
1809                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1810
1811                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1812                 shardManager.tell(new ActorInitialized(), respondActor);
1813                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1814                         DataStoreVersions.CURRENT_VERSION), getRef());
1815                 shardManager.tell(
1816                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1817                         respondActor);
1818
1819                 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1820                 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1821                         RemoveServer.class);
1822                 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1823                         removeServer.getServerId());
1824                 expectMsgClass(duration("5 seconds"), Success.class);
1825             }
1826         };
1827     }
1828
1829     @Test
1830     public void testRemoveShardReplicaRemote() {
1831         MockConfiguration mockConfig = new MockConfiguration(
1832                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1833                         .put("astronauts", Arrays.asList("member-1")).build());
1834
1835         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1836
1837         // Create an ActorSystem ShardManager actor for member-1.
1838         final ActorSystem system1 = newActorSystem("Member1");
1839         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1840         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1841
1842         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1843                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1844                         new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1845                 shardManagerID);
1846
1847         // Create an ActorSystem ShardManager actor for member-2.
1848         final ActorSystem system2 = newActorSystem("Member2");
1849         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1850
1851         String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1852         String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1853         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1854                 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1855                         new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1856
1857         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1858
1859         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1860                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1861                         new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1862                 shardManagerID);
1863
1864         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1865         //    akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1866         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1867         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1868         // look like so,
1869         //    akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1870         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1871         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1872         // dead letters.
1873         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1874         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1875         // thing works as expected
1876         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1877                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1878                         "member-2-shard-default-" + shardMrgIDSuffix);
1879
1880         LOG.error("Forwarding actor : {}", actorRef);
1881
1882         new TestKit(system1) {
1883             {
1884                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1885                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1886
1887                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1888                 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1889
1890                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1891                 leaderShardManager.tell(
1892                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1893                         mockShardLeaderActor);
1894                 leaderShardManager.tell(
1895                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1896                         mockShardLeaderActor);
1897
1898                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1899                 newReplicaShardManager.tell(
1900                         new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1901                         mockShardActor);
1902                 newReplicaShardManager.tell(
1903                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1904                         mockShardActor);
1905
1906                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1907                 leaderShardManager.underlyingActor().waitForMemberUp();
1908
1909                 // construct a mock response message
1910                 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1911                 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1912                         RemoveServer.class);
1913                 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1914                 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1915                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1916             }
1917         };
1918     }
1919
1920     @Test
1921     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() {
1922         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1923                 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1924     }
1925
1926     @Test
1927     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() {
1928         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1929                 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1930     }
1931
1932
1933     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1934                                                       final Class<?> firstForwardedServerChangeClass,
1935                                                       final Object secondServerChange) {
1936         new TestKit(getSystem()) {
1937             {
1938                 TestKit mockShardLeaderKit = new TestKit(getSystem());
1939                 final TestKit secondRequestKit = new TestKit(getSystem());
1940
1941                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1942                         .put(shardName, Arrays.asList("member-2")).build());
1943
1944                 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1945                         newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1946                                 .cluster(new MockClusterWrapper()).props()
1947                                 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1948                         shardMgrID);
1949
1950                 shardManager.underlyingActor()
1951                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1952
1953                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1954
1955                 shardManager.tell(firstServerChange, getRef());
1956
1957                 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1958
1959                 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1960
1961                 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1962             }
1963         };
1964     }
1965
1966     @Test
1967     public void testServerRemovedShardActorNotRunning() {
1968         LOG.info("testServerRemovedShardActorNotRunning starting");
1969         new TestKit(getSystem()) {
1970             {
1971                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1972                         .put("default", Arrays.asList("member-1", "member-2"))
1973                         .put("astronauts", Arrays.asList("member-2"))
1974                         .put("people", Arrays.asList("member-1", "member-2")).build());
1975
1976                 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1977                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1978
1979                 shardManager.underlyingActor().waitForRecoveryComplete();
1980                 shardManager.tell(new FindLocalShard("people", false), getRef());
1981                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1982
1983                 shardManager.tell(new FindLocalShard("default", false), getRef());
1984                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1985
1986                 // Removed the default shard replica from member-1
1987                 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1988                 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1989                         .build();
1990                 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1991
1992                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1993             }
1994         };
1995
1996         LOG.info("testServerRemovedShardActorNotRunning ending");
1997     }
1998
1999     @Test
2000     public void testServerRemovedShardActorRunning() {
2001         LOG.info("testServerRemovedShardActorRunning starting");
2002         new TestKit(getSystem()) {
2003             {
2004                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2005                         .put("default", Arrays.asList("member-1", "member-2"))
2006                         .put("astronauts", Arrays.asList("member-2"))
2007                         .put("people", Arrays.asList("member-1", "member-2")).build());
2008
2009                 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
2010                 ActorRef shard = actorFactory.createActor(MessageCollectorActor.props(), shardId);
2011
2012                 TestActorRef<TestShardManager> shardManager = actorFactory
2013                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props()
2014                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2015
2016                 shardManager.underlyingActor().waitForRecoveryComplete();
2017
2018                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2019                 shardManager.tell(new ActorInitialized(), shard);
2020
2021                 waitForShardInitialized(shardManager, "people", this);
2022                 waitForShardInitialized(shardManager, "default", this);
2023
2024                 // Removed the default shard replica from member-1
2025                 shardManager.tell(new ServerRemoved(shardId), getRef());
2026
2027                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
2028
2029                 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
2030             }
2031         };
2032
2033         LOG.info("testServerRemovedShardActorRunning ending");
2034     }
2035
2036     @Test
2037     public void testShardPersistenceWithRestoredData() {
2038         LOG.info("testShardPersistenceWithRestoredData starting");
2039         new TestKit(getSystem()) {
2040             {
2041                 MockConfiguration mockConfig =
2042                     new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2043                             .put("default", Arrays.asList("member-1", "member-2"))
2044                             .put("astronauts", Arrays.asList("member-2"))
2045                             .put("people", Arrays.asList("member-1", "member-2")).build());
2046                 String[] restoredShards = {"default", "astronauts"};
2047                 ShardManagerSnapshot snapshot =
2048                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
2049                 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
2050
2051                 // create shardManager to come up with restored data
2052                 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
2053                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
2054
2055                 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
2056
2057                 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
2058                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
2059                 assertEquals("for uninitialized shard", "people", notFound.getShardName());
2060
2061                 // Verify a local shard is created for the restored shards,
2062                 // although we expect a NotInitializedException for the shards
2063                 // as the actor initialization
2064                 // message is not sent for them
2065                 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
2066                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2067
2068                 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
2069                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2070             }
2071         };
2072
2073         LOG.info("testShardPersistenceWithRestoredData ending");
2074     }
2075
2076     @Test
2077     public void testShutDown() throws Exception {
2078         LOG.info("testShutDown starting");
2079         new TestKit(getSystem()) {
2080             {
2081                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2082                         .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
2083
2084                 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
2085                 ActorRef shard1 = actorFactory.createActor(MessageCollectorActor.props(), shardId1);
2086
2087                 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
2088                 ActorRef shard2 = actorFactory.createActor(MessageCollectorActor.props(), shardId2);
2089
2090                 ActorRef shardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig)
2091                         .addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
2092
2093                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2094                 shardManager.tell(new ActorInitialized(), shard1);
2095                 shardManager.tell(new ActorInitialized(), shard2);
2096
2097                 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2098                 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2099
2100                 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2101                 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2102
2103                 try {
2104                     Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2105                     fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2106                 } catch (TimeoutException e) {
2107                     // expected
2108                 }
2109
2110                 actorFactory.killActor(shard1, this);
2111                 actorFactory.killActor(shard2, this);
2112
2113                 Boolean stopped = Await.result(stopFuture, duration);
2114                 assertEquals("Stopped", Boolean.TRUE, stopped);
2115             }
2116         };
2117
2118         LOG.info("testShutDown ending");
2119     }
2120
2121     @Test
2122     public void testChangeServersVotingStatus() {
2123         new TestKit(getSystem()) {
2124             {
2125                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2126
2127                 ActorRef respondActor = actorFactory
2128                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2129                                 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2130
2131                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2132
2133                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2134                 shardManager.tell(new ActorInitialized(), respondActor);
2135                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2136                         DataStoreVersions.CURRENT_VERSION), getRef());
2137                 shardManager.tell(
2138                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2139                         respondActor);
2140
2141                 shardManager.tell(
2142                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2143                         getRef());
2144
2145                 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2146                         .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2147                 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2148                         ImmutableMap.of(ShardIdentifier
2149                                 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2150                                 Boolean.TRUE));
2151
2152                 expectMsgClass(duration("5 seconds"), Success.class);
2153             }
2154         };
2155     }
2156
2157     @Test
2158     public void testChangeServersVotingStatusWithNoLeader() {
2159         new TestKit(getSystem()) {
2160             {
2161                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2162
2163                 ActorRef respondActor = actorFactory
2164                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2165                                 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2166
2167                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2168
2169                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2170                 shardManager.tell(new ActorInitialized(), respondActor);
2171                 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2172
2173                 shardManager.tell(
2174                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2175                         getRef());
2176
2177                 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2178
2179                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2180                 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2181             }
2182         };
2183     }
2184
2185     public static class TestShardManager extends ShardManager {
2186         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2187         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2188         private ShardManagerSnapshot snapshot;
2189         private final Map<String, ActorRef> shardActors;
2190         private final ActorRef shardActor;
2191         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2192         private CountDownLatch memberUpReceived = new CountDownLatch(1);
2193         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2194         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2195         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2196         private volatile MessageInterceptor messageInterceptor;
2197
2198         TestShardManager(final Builder builder) {
2199             super(builder);
2200             shardActor = builder.shardActor;
2201             shardActors = builder.shardActors;
2202         }
2203
2204         @Override
2205         protected void handleRecover(final Object message) throws Exception {
2206             try {
2207                 super.handleRecover(message);
2208             } finally {
2209                 if (message instanceof RecoveryCompleted) {
2210                     recoveryComplete.countDown();
2211                 }
2212             }
2213         }
2214
2215         private void countDownIfOther(final Member member, final CountDownLatch latch) {
2216             if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2217                 latch.countDown();
2218             }
2219         }
2220
2221         @Override
2222         public void handleCommand(final Object message) throws Exception {
2223             try {
2224                 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2225                     getSender().tell(messageInterceptor.apply(message), getSelf());
2226                 } else {
2227                     super.handleCommand(message);
2228                 }
2229             } finally {
2230                 if (message instanceof FindPrimary) {
2231                     findPrimaryMessageReceived.countDown();
2232                 } else if (message instanceof ClusterEvent.MemberUp) {
2233                     countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2234                 } else if (message instanceof ClusterEvent.MemberRemoved) {
2235                     countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2236                 } else if (message instanceof ClusterEvent.UnreachableMember) {
2237                     countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2238                 } else if (message instanceof ClusterEvent.ReachableMember) {
2239                     countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2240                 }
2241             }
2242         }
2243
2244         void setMessageInterceptor(final MessageInterceptor messageInterceptor) {
2245             this.messageInterceptor = messageInterceptor;
2246         }
2247
2248         void waitForRecoveryComplete() {
2249             assertEquals("Recovery complete", true,
2250                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2251         }
2252
2253         public void waitForMemberUp() {
2254             assertEquals("MemberUp received", true,
2255                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2256             memberUpReceived = new CountDownLatch(1);
2257         }
2258
2259         void waitForMemberRemoved() {
2260             assertEquals("MemberRemoved received", true,
2261                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2262             memberRemovedReceived = new CountDownLatch(1);
2263         }
2264
2265         void waitForUnreachableMember() {
2266             assertEquals("UnreachableMember received", true,
2267                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2268                 ));
2269             memberUnreachableReceived = new CountDownLatch(1);
2270         }
2271
2272         void waitForReachableMember() {
2273             assertEquals("ReachableMember received", true,
2274                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2275             memberReachableReceived = new CountDownLatch(1);
2276         }
2277
2278         void verifyFindPrimary() {
2279             assertEquals("FindPrimary received", true,
2280                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2281             findPrimaryMessageReceived = new CountDownLatch(1);
2282         }
2283
2284         public static Builder builder(final DatastoreContext.Builder datastoreContextBuilder) {
2285             return new Builder(datastoreContextBuilder);
2286         }
2287
2288         public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2289             private ActorRef shardActor;
2290             private final Map<String, ActorRef> shardActors = new HashMap<>();
2291
2292             Builder(final DatastoreContext.Builder datastoreContextBuilder) {
2293                 super(TestShardManager.class);
2294                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2295             }
2296
2297             Builder shardActor(final ActorRef newShardActor) {